{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question parser\n",
    "\n",
    "Parse question into tokens assigned to 4 groups: \n",
    "* entities and predicates for the 1st hop (E1 P1)  \n",
    "* entities and predicates for the 2nd hop (E2 P2)  \n",
    "\n",
    "Then train a supervised sequence tagging model to extract the mention text spans.\n",
    "\n",
    "Reference: https://www.depends-on-the-definition.com/sequence-tagging-lstm-crf/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup\n",
    "dataset_name = 'lcquad'\n",
    "embeddings_choice = 'glove6B100d'\n",
    "\n",
    "import os\n",
    "os.chdir('/home/zola/Projects/KBQA/src')  # path to working directory for saving models\n",
    "\n",
    "# connect to MongoDB (27017 is the default port) to access the dataset\n",
    "# sudo service mongod start \n",
    "from pymongo import MongoClient\n",
    "\n",
    "\n",
    "class Mongo_Connector():\n",
    "    '''\n",
    "    Wrapper class for some of the pymongo functions: http://api.mongodb.com/python/current/tutorial.html\n",
    "    '''\n",
    "\n",
    "    def __init__(self, db_name, col_name):\n",
    "        # spin up database\n",
    "        self.mongo_client = MongoClient()\n",
    "        self.col = self.mongo_client[db_name][col_name]\n",
    "        \n",
    "    def get_sample(self, limit=100):\n",
    "        '''\n",
    "        Set limit to None to get all docs\n",
    "        '''\n",
    "        cursor = self.col.find()\n",
    "        if limit:\n",
    "            cursor = cursor.limit(limit)\n",
    "        return cursor\n",
    "\n",
    "\n",
    "mongo = Mongo_Connector('kbqa', dataset_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare dataset\n",
    "\n",
    "Encode questions with IO tokens: E1E2P1P2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 2574 lcquad questions\n"
     ]
    }
   ],
   "source": [
    "limit = None\n",
    "verbose = False\n",
    "\n",
    "tags = ['O', 'E', 'P']  # + p_tags\n",
    "modelname = 'joint'  # joint predicate\n",
    "\n",
    "import re\n",
    "from keras.preprocessing.text import text_to_word_sequence\n",
    "\n",
    "def parse_question(question_o, spans):\n",
    "    # exclude duplicate spans\n",
    "    spans = list(set(spans))\n",
    "    # sort the mention spans into the correct sequence order in the question\n",
    "    spans.sort(key=lambda tup: tup[0])\n",
    "    # parse question into mention spans\n",
    "    cursor_idx = 0\n",
    "    words, ye = [], []\n",
    "    for span in spans:\n",
    "        idx_start, idx_end, tag = span\n",
    "        # spacing\n",
    "        space = text_to_word_sequence(question_o[cursor_idx:idx_start])\n",
    "        words.extend(space)\n",
    "        ye.extend([''] * len(space))\n",
    "        # mention\n",
    "        mention = text_to_word_sequence(question_o[idx_start:idx_end])\n",
    "        words.extend(mention)\n",
    "        ye.extend([tag] * len(mention))\n",
    "        # move cursor to the end of the current annotation span\n",
    "        cursor_idx = idx_end\n",
    "    # add the remaining spacing\n",
    "    space = text_to_word_sequence(question_o[cursor_idx:])\n",
    "    words.extend(space)\n",
    "    ye.extend([''] * len(space))\n",
    "    if verbose:\n",
    "        print(spans)\n",
    "        print(words)\n",
    "        print(ye)\n",
    "    assert len(words) == len(ye)\n",
    "    return words, ye\n",
    "\n",
    "\n",
    "samples = mongo.get_sample(limit=limit)\n",
    "# iterate over the cursor\n",
    "questions = []\n",
    "n_words_distr = []\n",
    "labels = []\n",
    "count = 0\n",
    "question_types = []  # indicate complex questions by number of hops\n",
    "for doc in samples:\n",
    "    _complex = 0  # default indicator for simple questions\n",
    "    # get sample annotations\n",
    "    question_o = doc['question']\n",
    "    # get URI distribution across the hops\n",
    "    e1, p1 = doc['1hop']\n",
    "    e2, p2 = doc['2hop']\n",
    "    \n",
    "    # collect span annotations\n",
    "    e_spans = []\n",
    "    p_spans = []\n",
    "    # fix: some entities are erroneously annotated as predicates\n",
    "    for e in doc['entity mapping']+doc['predicate mapping']:\n",
    "        # fix: use labels to find substrings since some of the span indices annotations are incorrect\n",
    "        matches = re.finditer(re.escape(e['label'].lower()), question_o.lower())\n",
    "        # pick tag for this mention\n",
    "        if e['uri'] in e1:\n",
    "            e_spans.extend([(match.start(), match.end(), 'E1') for match in matches])\n",
    "        elif e['uri'] in e2:\n",
    "            e_spans.extend([(match.start(), match.end(), 'E2') for match in matches])\n",
    "            _complex = 1  # indicate complex question\n",
    "        elif e['uri'] in p1:\n",
    "            p_spans.extend([(match.start(), match.end(), 'P1') for match in matches])\n",
    "        elif e['uri'] in p2:\n",
    "            p_spans.extend([(match.start(), match.end(), 'P2') for match in matches])\n",
    "            _complex = 1  # indicate complex question\n",
    "\n",
    "    if modelname == 'entity':\n",
    "        words, _y = parse_question(question_o, e_spans)\n",
    "    elif modelname == 'predicate':\n",
    "        words, _y = parse_question(question_o, p_spans)\n",
    "    elif modelname == 'joint':\n",
    "        words_e, y_e = parse_question(question_o, e_spans)\n",
    "        words_p, y_p = parse_question(question_o, p_spans)\n",
    "        if words_e == words_p:\n",
    "            words = words_e\n",
    "            _y = []\n",
    "            for tag_e, tag_p in zip(y_e, y_p):\n",
    "                if tag_e == '' or tag_p == '':\n",
    "                    _y.append(tag_e+tag_p)\n",
    "                else:\n",
    "                    _y.append(tag_e) # use the entity tag when tags overlap\n",
    "                \n",
    "    #     add the sample to the dataset \n",
    "    questions.append(words)\n",
    "    n_words_distr.append(len(words))\n",
    "    labels.append(_y)\n",
    "    question_types.append(_complex)\n",
    "    count += 1\n",
    "\n",
    "dataset_size = len(questions)\n",
    "print(\"Loaded %d %s questions\"%(dataset_size, dataset_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['what', 'is', 'the', 'company', 'of', 'ford', 'theatre']\n",
      "['', '', '', 'P1', '', 'E1', 'E1']\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# show sample question\n",
    "i = 200\n",
    "print(questions[i])\n",
    "print(labels[i])\n",
    "print(question_types[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique words 3843\n",
      "Maximum question length in the dataset: 24\n",
      "366 OOV words\n",
      "Model settings saved.\n"
     ]
    }
   ],
   "source": [
    "# load pre-trained word embeddings for question semantic representation\n",
    "from pymagnitude import *\n",
    "embeddings_path = \"/home/zola/Projects/KBQA/data/embeddings/\"\n",
    "embeddings = {'glove6B100d': \"glove.6B.100d.magnitude\"}\n",
    "vectors = Magnitude(embeddings_path + embeddings[embeddings_choice])\n",
    "\n",
    "words = list(set([word for q in questions for word in q]))\n",
    "n_words = len(words)\n",
    "print(\"Number of unique words %d\"%len(words))\n",
    "word2idx = {w: i + 1 for i, w in enumerate(words)}\n",
    "\n",
    "# dataset parameters for training the model\n",
    "max_len = max(n_words_distr)\n",
    "print(\"Maximum question length in the dataset: %d\"%max_len)\n",
    "\n",
    "# prepare data and pad the max length with 0s\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "X = [[word2idx[w] for w in s] for s in questions]\n",
    "X = pad_sequences(maxlen=max_len, sequences=X, padding=\"post\", value=0)\n",
    "\n",
    "# encode tags\n",
    "n_tags = len(tags)\n",
    "y = [[tags.index(tag[0]) if tag else 0 for tag in s] for s in labels]\n",
    "y = pad_sequences(maxlen=max_len, sequences=y, padding=\"post\", value=0)\n",
    "from keras.utils import to_categorical\n",
    "y = [to_categorical(i, num_classes=n_tags) for i in y]\n",
    "\n",
    "# load embeddings into matrix\n",
    "import math\n",
    "word_embedding_matrix = np.zeros((n_words+1, vectors.dim))\n",
    "\n",
    "n_oov = 0\n",
    "\n",
    "for w in word2idx:\n",
    "    # get the word vector from the embedding model\n",
    "    if w in vectors:\n",
    "        word_vector = vectors.query(w)\n",
    "    # OOV word\n",
    "    else:\n",
    "        n_oov += 1\n",
    "        word_vector = vectors.query('unk')\n",
    "    word_embedding_matrix[word2idx[w]] = word_vector\n",
    "\n",
    "# loaded vector # may be lower than total vocab due to w2v settings\n",
    "print('%d OOV words'%n_oov)\n",
    "\n",
    "model_settings = {'embeddings': word_embedding_matrix, 'word2idx': word2idx,\n",
    "                  'max_len': max_len, 'n_words': n_words, 'n_tags': n_tags, 'emb_dim': vectors.dim}\n",
    "\n",
    "# save model settings\n",
    "import pickle as pkl\n",
    "f = open('%s_%s.pkl'%(dataset_name, embeddings_choice), 'wb')\n",
    "pkl.dump(model_settings, f, -1)\n",
    "f.close()\n",
    "print(\"Model settings saved.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mention extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on 2316 samples testing on 258 samples\n"
     ]
    }
   ],
   "source": [
    "# split dataset into training and testing subsets\n",
    "test_size = 0.1\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "# fix random seed\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size,\n",
    "                                                    stratify=question_types, random_state=103232)\n",
    "print(\"Training on %d samples testing on %d samples\" % (len(X_train), len(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zola/anaconda3/envs/tf36/lib/python3.6/site-packages/keras_contrib/layers/crf.py:314: UserWarning: CRF.loss_function is deprecated and it might be removed in the future. Please use losses.crf_loss instead.\n",
      "  warnings.warn('CRF.loss_function is deprecated and it might be removed in the future. Please '\n",
      "/home/zola/anaconda3/envs/tf36/lib/python3.6/site-packages/keras_contrib/layers/crf.py:320: UserWarning: CRF.accuracy is deprecated and it might be removed in the future. Please use metrics.crf_accuracy\n",
      "  warnings.warn('CRF.accuracy is deprecated and it might be removed in the future. Please '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_16 (InputLayer)        (None, 24)                0         \n",
      "_________________________________________________________________\n",
      "embedding_16 (Embedding)     (None, 24, 100)           384400    \n",
      "_________________________________________________________________\n",
      "bidirectional_16 (Bidirectio (None, 24, 100)           60400     \n",
      "_________________________________________________________________\n",
      "time_distributed_14 (TimeDis (None, 24, 50)            5050      \n",
      "_________________________________________________________________\n",
      "crf_14 (CRF)                 (None, 24, 3)             168       \n",
      "=================================================================\n",
      "Total params: 450,018\n",
      "Trainable params: 65,618\n",
      "Non-trainable params: 384,400\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# bi-LSTM CRF model architecture\n",
    "from keras.models import Model, Input\n",
    "from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional\n",
    "from keras_contrib.layers import CRF\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "def build_model(model_settings):\n",
    "    # architecture\n",
    "    input = Input(shape=(model_settings['max_len'],))\n",
    "    model = Embedding(input_dim=model_settings['n_words']+1, output_dim=model_settings['emb_dim'],\n",
    "                      weights=[model_settings['embeddings']],\n",
    "                      input_length=model_settings['max_len'], mask_zero=True, trainable=False)(input)\n",
    "    model = Bidirectional(LSTM(units=50, return_sequences=True,\n",
    "                               recurrent_dropout=0.1))(model)  # variational biLSTM\n",
    "    model = TimeDistributed(Dense(50, activation=\"relu\"))(model)  # a dense layer as suggested by neuralNer\n",
    "    crf = CRF(model_settings['n_tags'])  # CRF layer\n",
    "    out = crf(model)  # output\n",
    "    model = Model(input, out)\n",
    "    model.compile(optimizer=Adam(lr=0.0001), loss=crf.loss_function, metrics=[crf.accuracy])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "# load model settings\n",
    "import pickle as pkl\n",
    "with open('%s_%s.pkl'%(dataset_name, embeddings_choice), 'rb') as f:\n",
    "    model_settings = pkl.load(f)\n",
    "model = build_model(model_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2084 samples, validate on 232 samples\n",
      "Epoch 1/50\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-234-14973fa2cd1f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m log = model.fit(X_train, np.array(y_train), batch_size=32, epochs=50,\n\u001b[1;32m     11\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m                 validation_split=0.1, verbose=1)\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2695\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2696\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2697\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_make_callable_from_options'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2698\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_sparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2699\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m()\u001b[0m\n\u001b[1;32m    204\u001b[0m                     \u001b[0mv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_keras_initialized\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0muninitialized_vars\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m                     \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muninitialized_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m     \u001b[0;31m# hack for list_devices() function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;31m# list_devices() function is not available under tensorflow r1.3.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    927\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 929\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    930\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1150\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1152\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1153\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1328\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1329\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1332\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1334\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1335\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       \u001b[0;31m# Ensure any changes to the graph are reflected in the runtime.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1317\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m   1319\u001b[0m           options, feed_dict, fetch_list, target_list, run_metadata)\n",
      "\u001b[0;32m~/anaconda3/envs/tf36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_extend_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1350\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1351\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session_run_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1352\u001b[0;31m       \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExtendSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;31m# The threshold to run garbage collection to delete dead tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# callbacks\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping, TerminateOnNaN, ModelCheckpoint\n",
    "cb_redlr = ReduceLROnPlateau(monitor='val_crf_viterbi_accuracy', factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n",
    "cb_early = EarlyStopping(monitor='val_crf_viterbi_accuracy', min_delta=0, patience=5, verbose=1)\n",
    "cb_chkpt = ModelCheckpoint('checkpoints/_'+modelname+'{epoch:02d}-{val_crf_viterbi_accuracy:.2f}.h5', verbose=1, save_best_only=True, save_weights_only=True, period=5)\n",
    "\n",
    "callbacks_list=[cb_redlr, cb_early, cb_chkpt]\n",
    "\n",
    "# start training\n",
    "log = model.fit(X_train, np.array(y_train), batch_size=32, epochs=50,\n",
    "                callbacks=callbacks_list,\n",
    "                validation_split=0.1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAKvCAYAAABpkwknAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xl8VOXd///XmSUz2VcghE1kURCRfXMFAwgK4oKorUvVti71tnrXtvrVn7a3ttpWb2tva6212t2tWlypRcEFFBAFZZFdSci+ZyaZ9Vy/PyYEkASiJJmEvJ+PRzhnzlxz5jNcAd6cXOe6LGOMQURERESkh3PEuwARERERka5AwVhEREREBAVjERERERFAwVhEREREBFAwFhEREREBFIxFRERERAAFYxERERERQMFYRERERARQMBYRERERARSMRUREREQAcMXzzYuKiuLyvjk5OVRUVMTlvaXzqb97FvV3z6L+7lnU3z1Pe/V5Xl5em9rpirGIiIiICArGIiIiIiKAgrGIiIiICBDnMcZfZowhEAhg2zaWZXXY+5SWlhIMBjvs/NJ+jDE4HA68Xm+Hfk+IiIiIdKlgHAgEcLvduFwdW5bL5cLpdHboe0j7iUQiBAIBEhMT412KiIiIHMW61FAK27Y7PBRL9+NyubBtO95liIiIyFGuSwVj/ahcWqPvDREREeloXSoYi4iIiIjEi8Yt7KeqqopFixYBUF5ejtPpJCsrC4BXX32VhISEw57j5ptv5oYbbmDo0KGttnnqqadIS0vj/PPPb5/CRUREROSIKRjvJysri//85z8APPDAAyQnJ3Pttdce0MYY0zxTQkv+93//97Dvc+WVVx5xrZ0tEolo/LeIiIgc1TSUog127drFjBkz+NGPfsTs2bMpLS3lhz/8IXPmzGH69OkHhOEFCxawYcMGIpEII0aM4Gc/+xn5+fnMmzeveUnD+++/n8cff7y5/c9+9jPOPvtsTj31VNasWQNAQ0MD3/72t8nPz+f6669nzpw5bNiw4aDafvWrXzF37tzm+owxAOzYsYOFCxeSn5/P7NmzKSgoAODhhx/mzDPPJD8/n/vuu++AmgHKyso4+eSTAfj73//Oddddx+WXX843v/lN6uvrWbhwIbNnzyY/P7/5PxEAzzzzDPn5+eTn53PzzTdTW1vL1KlTiUQiANTW1jJlyhSi0Wj7dYyIiIhIO+qylwDtpx/HFOxq13NaAwbjuPjbX+u1W7du5cEHH+T+++8H4LbbbiMzM5NIJMLChQs5++yzGT58+AGvqaurY8qUKdx+++3cfffdPP3003zve9876NzGGF599VXeeOMNHnroIf72t7/xxz/+kV69evH444+zceNGzjrrrBbruvrqq/nBD36AMYYbbriBZcuWMWPGDG644QZuueUWZs2aRSAQwBjDG2+8wbJly3jllVdITEykurr6sJ977dq1vPHGG2RkZBAOh3nyySdJSUmhoqKCc889l5kzZ7Jx40YeeeQRFi9eTGZmJtXV1aSnpzNhwgSWLVvGzJkzefHFF5k3b56myRMREZEuq8sG465m0KBBjBkzpvnx4sWL+cc//kE0GqWkpIStW7ceFIy9Xi8zZswAYPTo0axatarFc8+ZMweAE088sfnK7urVq7nhhhsAOOGEEzjuuONafO17773H7373O4LBIFVVVYwePZpx48ZRVVXFrFmzmuvY2/biiy9ung84MzPzsJ/79NNPJyMjA4gF+HvvvZc1a9ZgWRbFxcVUVVWxYsUK5s+f33y+vdtLL72UP/7xj8ycOZNnn32Whx9++LDvJyIiIhIvXTYYf90rux0lKSmpeX/nzp384Q9/4NVXXyU9PZ0bb7yxxZX09r9Zz+l0tjqMYG+7/dvsHRJxKI2Njdxxxx0sWbKEvn37cv/99xMIBICWpzdr7Zwul6v5uS9/jv0X1Xj++eepr69nyZIluFwuxo8f33w1uqX3mzp1KnfccQcrVqzA5XId8oZEERERkXjTGOOvwefzkZKSQmpqKqWlpSxfvrzd32PSpEm8/PLLAGzevJmtW7ce1KaxsRGHw0FWVhY+n4/XXnsNgIyMDLKysnjjjTeA2IqCjY2NnHbaaTz99NM0NjYCNA+l6N+/P5988gkQm32jNXV1dWRnZ+NyuXjnnXcoKSkB4NRTT2Xx4sXN59t/iMb555/PjTfe2Dzbh4iIiEhXpWD8NZx44okMGzaMGTNmcOuttzJx4sR2f4+rrrqKkpIS8vPzeeyxxzjuuONIS0s7oE1WVhYLFy5kxowZXH311YwdO7b5ud/85jf8/ve/Jz8/n/POO4/KykpmzpzJGWecwdy5c5k5c2bzDYDXXXcdTzzxBPPnz6empqbVmi688ELWrl3LnDlzeOWVVxg8eDAAI0eO5Prrr+eCCy5g5syZ3HPPPc2vOe+886irq2P+/Pnt+dsjIiIi0u4s05af2XeQoqKiAx43NDQcMGSho7hcrubZErqqSCRCJBLB6/Wyc+dOLr30Ut57771uN2Xa4sWLWb58eZumsTuUI/neyMnJaZ4RRI5+6u+eRf3ds6i/e5726vO8vLw2teteKasH8fv9LFq0qDnA33///d0uFP/4xz/m3Xff5W9/+1u8SxERERE5rO6VtHqQ9PR0lixZEu8yjsjeeZJFREREugONMRYRERERQcFYRERERARQMBYRERERARSMRUREREQABeMDXHjhhQct1vH4449z2223HfJ1w4YNA6CkpIRvf7vlFfsuvPBC1q9ff8jzPP74482LbwBcdtll1NbWtqFyERERETlSCsb7Offcc1m8ePEBxxYvXsyCBQva9Prc3NzmRTO+jj/84Q8HBOO//OUvpKenf+3zdTZjDLZtx7sMERERka9FwXg/Z599NkuXLiUYDAJQUFBAaWkpkyZNwu/3c9FFFzF79mzOPPNM/v3vfx/0+oKCAmbMmAHElmu+7rrryM/P59prryUQCDS3+/GPf8ycOXOYPn06v/rVrwB44oknKC0tZeHChVx44YUATJ48maqqKgAee+wxZsyYwYwZM5rDd0FBAaeffjq33nor06dP55JLLjkgWO/1xhtvcM455zBr1iwWLVpEeXk5EJsr+eabb+bMM88kPz+/eTnoZcuWMXv2bPLz87nooosAeOCBB/jd737XfM4ZM2ZQUFDQXMNtt93G7NmzKSoqavHzAaxbt4758+eTn5/P2Wefjc/n47zzzmPDhg3Nbc4991w2bdr0lfpNREREpD102XmM//BhKbuqA4dv+BUMzvRyzYQ+rT6flZXFmDFjWL58ObNnz2bx4sXMnz8fy7LweDw88cQTpKamUlVVxbx585g1axaWZbV4rj//+c8kJiaydOlSNm3axFlnndX83I9+9CMyMzOJRqMsWrSITZs2cfXVV/P73/+e5557jqysrAPO9cknn/Dss8/yyiuvYIzhnHPOYerUqaSnp7Nr1y4eeeQRfvnLX/Ld736X1157jQsuuOCA10+aNImXX34Zy7L4+9//zm9/+1vuuusuHnroIVJTU3nzzTcBqKmpobKykltvvZUXXniBgQMHUl1dfdjf1x07dvDggw/y85//vNXPN3ToUK677joeffRRxowZQ319PV6vl0suuYRnn32WUaNGsWPHDkKhECNHjjzse4qIiIi0ty4bjONlwYIFLF68uDkYP/jgg0BsmMB9993HqlWrsCyLkpISysvL6d27d4vnWbVqFVdddRUAI0eOZMSIEc3Pvfzyy/ztb38jGo1SWlrKtm3bDhkGV69ezVlnndW8JPKcOXNYtWoVs2bNYsCAAYwaNQqA0aNHU1BQcNDri4uLue666ygrKyMUCjFw4EAA3n33XX772982t8vIyOCNN95gypQpzW0yMzMP+3vWv39/xo8ff8jPZ1kWvXv3ZsyYMQCkpqYCMG/ePH79619z55138swzzzRfoRYREZHOY4yBcAgCDRBojH01xrZm/2N7v4JNP6F2usDlim3d7n2PXS5wupv2Y1vL6YKxU1q9qNgVdNlgfKgrux3prLPO4ic/+QmffvopgUCAE088EYAXXniByspKXn/9ddxuN5MnT24ectGaljp+9+7dPPbYY7z66qtkZGTw/e9//4BhFi0xxrT6nMfjad53Op0tnuvOO+/kO9/5DrNmzWLlypUHhP2WamzpmNPpPGD88P6ffW9gP9Tna+29EhMTOfXUU/n3v//Nyy+/zGuvvdbqZxUREZG2McbEwmtdLdTHvkx9LdTVHPh475evDqLRtp3ckwheL2BBNAyRSNNX+NA1ORw4H/vXkX+4DtRlg3G8JCcnM3XqVG655ZYDbrqrr68nJycHt9vNihUrKCwsPOR5Jk+ezIsvvsjJJ5/MZ599xubNm5vPk5iYSFpaGuXl5SxbtoypU6cCkJKSgs/nO2goxZQpU7j55pv53ve+hzGGJUuW8PDDD7f5M9XV1ZGbmwvAc88913z89NNP58knn+SnP/0pEBtKMX78eG6//XZ2797dPJQiMzOTAQMGsHTpUgA+/fRTdu/e3eJ7tfb5hg4dSmlpKevWrWPMmDH4fD68Xi8ul4tLL72UK6+8kkmTJrXpCrWIiEh3Y+xoLKRWV4KvFtPYAI0NsSu0DU3bRj+msREa/U1XbGPHCDaCASxiv1hW05YD97HA0fQ4GIhdAW6JNxFS0yEtA3L6YA0eDilpkJgUe86bhOVNbNqPPW7e93ixHC3fomaMiYXraFNQ/nJobmvwjiMF4xYsWLCAa665hkcffbT52Pnnn88VV1zBnDlzOOGEExg6dOghz3H55Zdzyy23kJ+fz8iRI5uHEJxwwgmMGjWK6dOnM3DgQCZOnNj8mm984xt885vfpHfv3jz//PPNx0888UQWLlzI2WefDcAll1zCqFGjWhw20ZL//u//5rvf/S65ubmMGzeu+XU33XQTt99+OzNmzMDhcHDLLbcwd+5cfvGLX3DNNddg2zY5OTk8/fTTzJ07l+eff56ZM2cyZswYjj322Bbfq7XPl5CQwKOPPsodd9xBIBDA6/XyzDPP4HK5GD16NCkpKSxatKhNn0dERKQrMaEg1FRCdRWmuiK2X1OFqa6E6gqoqYLaKmht5ibLAYmJkJgcC5+JyZCWgdW7b9MxbywAGwADxhy4D03HzL5jCV5IS4fUdKzUjOZ9UtOx3Akd8vtgWda+YRSew7fviixzqJ/Td7CioqIDHjc0NBzwY/mO4nK5iEQiHf4+0jYlJSVceOGFvPPOOzha+V/okXxv5OTkUFFRcSQlSjei/u5Z1N89y+H62xgDFaXwxXbMFzswu3cCBis9E9KzID0D0rOw0jIhPRMyMsGTeMgxr6axIRZuqysxNZX79qv37eOvP/iF3kTIyIbMbKymLRnZWJlZkJoBSclNV2iTYldhu/C423hqrz/jeXl5bWqnK8YSV8899xz3338/d911V6uhWERE5MsOCMGfb8fs3gFf7IAGX6yB0wV5A8DlxpTsgbrq2I/0abrYuleCJxaSm4KzleDF1FbFAm91RWxIw5elpkNmDmT3xho64ksBOAcys7C8HX+hT9qfgrHE1cKFC1m4cGG8yxARkS7I2FHw+2I3hvnqCXwWwt6wruUQ3G8Q1vhpMGgo1qAh0O8YLLd737mMiV3Zra2B2ipMXTXUVkNNNdRVY2qrYc9uTDAAGVnQtz/WyDH7XenN2be/33nl6NKlgnEcR3VIF6fvDRGRr8/UVsfGnaakYrk6J9QZY2I3XAUDEAxCKAihwH6PA5gGP/jqY8HXX4fZu19fFwuxDb59Y2iBWoiNX+13DNaEk2HQEKxBQyFv0GHDqmVZsRvMUtKg30A0cKHtjDGEogZ/2KYhFMUftvGHoridFqkJTlI9TlISnHhc3f8nv10qGDscDiKRCC5XlypL4iwSiWiYhYjIV2AiEdixGfPph5hP10LRfjMJJSbtC4jJqVh791NSITUNK7npcUJC85y1pqU5bAMNsePBwH7HA7EAvDf8mlZuNvsyd8K+GlLSYld8m/ZJjh23UtPIGDiYGm9yp4X7nmJrRSMrd9dTH4riD9n4w1EavrSNtKErE5wWqR7nAWE5zeMkJcERO+5xMn1wOk5H1/1vSZdKoF6vl0AgQDAY7NBB6B6P57BzEEvXYIzB4XDg9XrjXYqISJdmaioxn67FbFgLm9bFgqrTBcNGYk2bAR5v85AEfHUYX11sLtvigtixpgUbDvvzOcsRmyXBs/90XomQnYrlSQSPJ/ZeCZ7Yl8fb/Njabx+Ptymkp2N52jaFgTsnB0s3W7YL2xjW7vHz4uZKNpY14nJYpHmcJLkdJCc4Sfc66ZvqJjlh37Hk/bZJbgdh21AfilIfjOIL2s379aEovmCUgtogvqZjUQMuB5x5bHq8P/ohdalgbFkWiYmJHf4+uotZRES6OxONws4t+64KF+6KPZGZgzXpNKxR42HE6DbfBGbCYfDX7RvKEA4dGHz3BuEEj2ZQ6MbCUZvlu+r41+YqCutC5CS5uGpcb2YOTSfJ7eyQ9zTG0Bix8YfsLv+906WCsYiIiBzMNPihvATKizFlxVCwC7PpY2jwg9MJQ0ZgXXBFLAz3G/S1wofldsdmV8jI7oBPIPHmC0ZZsq2GV7ZUUR2IMjjTw83T+nLKoDRcHTy0wbIsktzODgve7UnBWERE5CsywSDUNy2tW1eL2W+fRl/s6mpScmxxhsQkrKSU2OPmY03Hm+6pMcbEZkgoL8GUF8dCcNne/eLYUIf9ZeZgjZ2KdeIEGHESVlJyHH4XpDso84V5aUsV/9leQyBiGJObxPdHZnNSblKXv3obDwrGIiLSY5hIuOkmsYNnSCAYiAXeA44HocGHqa+FuqbwW18be74lHm8s9IYCseV8m2ZUaHXc7t5xto0NB57TckBWDvTuizXuZOidi9UrF3r3hZzc2HK90q6MMWwqb2RnVYCRvZMYnOnB0Y2D486qAC9uquK93XVYwKmD0lgwMovBmbpn51AUjEVE5KhjQkEoKcQU7Yai3ZiigtjMDBWlB0z/dVgJnljQTU2HtPTYEr37La970FK7nn2hw9h2LOw2+GNXkRv80NgQGxax91hjQ2zfmwi9cmPn79UXsntp5oX9GGPYUhGgYHchQ1JgcGb7jXMORGze+byOV7dU83nNvhvz0z1OTuqbzNi+yYzpm0xWYsdHJmMMvpBNZUOYioYIFQ1hKhsiVDVGCEUNUdsQsZu2hn37TduoDRFjCEcNZf4wiS4H84/P4pzjMumVrO+ntlAwFhGRbssEg5jdOw4dgJ1O6J2HNXAITD4jNg1Y06wI1v4zJHx5352AdQRTRVoOR+xqcGIS0Gvf8SP7yD1KuT/Msl21LNtZS1F9uPl4XmoCpx6TyimD0hiY3rYZLb6suD7E61urWbqzFn/I5pgMDzdMzuWk3CQ2lTXycbGfdSV+3vm8DoBBGR7GNgXlkb0TSXB+te+NqG2oCUSoDUSp3C/07ttGqGwIE4we+B83hwXpXhcep4XLYeF0WLgc4LRij90OC6/LETvmsJqPzx2ewcyhGaQkdP1xvV2JZeK4ckJRUVFc3lezUvQs6u+eRf3dNRljYkHVtvfb2l96HI0Nc/D7wF+Padqy39bsXfRh73Ff3cEBOG8g5A3EyhsAeQNjxzQ/frsKR22K6sMU1AabvkKU+ML0T0tgRK9ETuidRP/0hK81FKExbPN+QT3LdtbyaWkDBhjVO5Hpx6Zzxoj+vLWpkHc/r2NDWQO2iQXWUwfFQnLf1IRDnts2hnXFfl7dUs3aIj8OC6YMSOWc4zIZ0SvxoKvQtjF8Xh2MheRiP5vKG4nYhgSnxQm9kxjbN5mTcpNwOCxqGiNUN0aoCUSpboxQHYjEjgWi1DRGqAtGDxpS47AgK9FFdpKbnCQXOUn77Se7yU5ykel1del5fztae/2dnpeX16Z2CsZy1FN/9yzq7/gyZUWY9Wsw61fDrq0QjYBt2r7QQ2sSkyApBZJTITkFq2mb1LcfjenZCsAdJBix2VMXYndT+N0XgkPYTenBYUGfFDe9kt3srglSE4gCkJrg4PheSYzslciI3okMzfLibuUqq20MG0obeGtnLe8X1BOIGHJT3Ew/Np3pg9PokxILvPv/+a5ujLBydz3vflHH5vLYHMzDsr2cMiiVkwemHTB0wBeK8tbOWl7bWk1xfZgMr5PZwzKYPTSD7KS2DzEIRGw2lDawrtjPx8V+CutCLbZzOywyE51keF1kJrqatvseZybGQnBGDw+9baFg3An0D2fPov7uWdTfnctEo7EV1tavwXyyBkoKY0/0G4R1/OjY0ATLAQ4HWNZ+W2csUVmOpm3TY28SVvK+AExyKiQmtxp41d/twxhDVWOEnVVBdlQH2FkV4IuaIKW+cPNVTqcFfVMTGJDuYUD6vm1eakLzUsDGGEp8YTaVNbCpvJFNZY0U1cfCY4LTYli2lxFNYfn4XonUBqLNQyXKGyIkuR2cPDCVGcemt3gFt7X+LveHee+LOt79op4dVbGbGEf2SmTawFQK60Is31VLIGI4LieRs4dnMG1gGm7nkQfScn+YjWUNOC0rFoATnWR6XSS5HZrxoZ0oGHcC/UXas6i/exb1d8czDT7Mxo9h/WrMho9iwxqcLjhuFNboSVijJ8RmUOgE6u+vzpjYjVk7qgLsqAqysyrAzupA85VeC8hLS+CYDA8DM/aF4L4pCV8rTNYEImwub2RzU1jeURXANrH3McT+P3RSbjIzjk1ncv+U5pDdkrb0d3F9iHe/qOO9z+v5ojaI22Fx2jFpzB2eydBszcjQ3XR2MNbPnEREBGgaBxwJx1Y8C4djU5XtfRwKYXZtjQ2R2L4JolFIScMaPRHrpIkwcixWYttWWJPOVROI8GlJA9urAuxoCsH+UGxoi8OCgekexuWlMCTLw5BML8dkekl0f/2bDr8sw+ti6oBUpg5IBWLDEbZWNLKpvBGvy+LUQWlfaTjD4fRNTeCiUTlcNCqHoroQKR4naR7dgCZto2AsInIUMuEw1FZBTSXUVGFqqqCmCmqb9murDwq+RMKHP3HeQKxZC7BGT4Jjh2M5FDi6mmDEZmNZA+tLGlhf4mdXdWwKMrfD4phMD6cMTGNIlpdjszwMyvB85dkVjpTX5WB0bjKjczt+UZK8tEPfjCfyZQrGIiLdmAmHMKvfgW2bME0hmNqqg1dKA3C5ID0LMrOhb//YVGXuhP2+3Pu2rn3HrIQEcLmhT79OGyLRVQQjNo1hG6/bgcdpdclxo1HbsLM6wPriBtaV+NncNHOCy2Exolcil53Ui5P6JjE409vhS/+KdHcKxiIi3ZCpq8Ysfx2z/PXYSmyp6ZDVK7ZIxLCRsQCckYWVkQ0ZsX2SU792sIvahqgxuI3pkuHwSPlDUQrr9s26sHdb7t9385lF7Gqn1+0g0WWR6HbgdTlIbDrmdTlIdDtITXAyuk8Sw3MSO2zGgZL6EOtLYkH40xI/9U1DIwZnejjnuExOyk3ihN5JhxyvKyIHUzAWEelGTOHnmKWLMavehkgETpyAY+a5cPzoDgms5f4wS7bV8J/tNdQGozgsSHA68LosPC4HXqeDBFdsgQGPM3bM44o9vzcoJrudJLkdJCU4SDrgcWzbWVcxo7ahLhhlz94A3LQtrA1R1Rhpbud2WPRPT+C4HC/5Q9JJSXDGrhw3fQXCNoGmK8mBiE11IEpjfZhAZN/xvxNbOW1i/xQm9U9hTG7yEYVUXyjKp03ThK0r9lPiiw17yU5yMal/KmP6JjO6TxIZnbA6m8jRTH+CRES6OGPbsGEt9tKXYPN6SPBgnTIT68x5WLn92/39bGNYX9LAa1ur+XCPD4CJ/VIYnp1IMGoTjNgEIqZ5PxgxsYAYjjbvB6OGQNgmbB9+4qMEp0Wy20Gi20lKgoNUj5OUhNh+isdJasLex05SPI7mx8kJzthqYo0RagL7FlKoCcQWWWjeNh2rC0bZvxyvy8GA9ATG9E1iQJqH/k2zL/ROdh/RlV5/KMraIj+rC+tZubuepTtqSXBajO2bzKT+KUzsl0K699D//EZsw7aKRj4u8bOuuIFtlY3YZu/43CTmH5/FSX2T6JeacFRewReJFwVjEZEuygQDmPffwrz5MpTsgYxsrPMvxzptdmyBi3bmC0Z5c2ctS7ZVU1QfJt3j5PyR2Zw1LOOAxRK+inDUpiEc+/KHbBrC0f0eR2kM2/jDseP+UGy/NhC7qusLxY4dKlpbbGnx+QSnRYbXSbrXRa9kN8NzvGR4Ywsq5KUl0D8tgZwkV4eEyuQEJ6cdk8Zpx6QRjho2ljWwqrCeVYU+VhX6cFhwfE4ik/qnMLl/KnlpCRhjKK4Ps64kdkX4k5IGGiM2DguGZnm58IRsxvZNZnhOosYJi3QgzWMsRz31d8/SlfrbGAOffYLZujF241uCJ3ZDW0ICuD2xm9rcbnB7mo4lsNnn4Jkdjdg1VaSVF5DeWENasof04ceRMXw46UkJpHtdpHucJCe0zyICO6oCvLa1mnc+ryMUNRyfk8jc4RlMG5ja6kplnSVqGxrCNr5QFF8oSn0wii/U9DgYxe1NJMEOkdG0qtjeFcYSXV1vgQVjDDurg6wqrGd1oa95toj+aQmEorG5hQF6J7sZ2zeZMX2TGN0nmRRNNdasK/35ls6heYxFRLo5Ew5jVr+DWboYCj9vvd1++3XuJP5y7Fze7DuJrGAtvQI17Mg4htreyTRELSgHyksOeL3LAameWEhO8+4bchAbirBvSEJqgpOU/Y4lOB2EojYrvqjn9W3VbKkI4HFaTB+czlnDMjg2q+ssguB0WKR6Yp+pJd0pKFmWxZAsL0OyvFw6uhelvhCrC318uMeHx+XgvJFZjO2bTG6Ku8uFepGeQsFYRKSdmPpazNuvY5a9BnU1sTl/r7gRa9JpsWWQQ6GmxTOa5g0OB7GDId4si/LnPS4aohYL0uq5KLWWpBNOap4aLRy1qQ1GqQtEqQ1GqQ1EqA00bYNR6oJRagNRqhqC1DddSY0e4meBCU4LhwWBiCEvNYFrxvdm+rGxm8yk8/RJSWDe8VnMOz4r3qWISBMFYxGRI2SKdmOWvoT5YHks9I4aj2Prx1gzAAAgAElEQVTmfBgx5sArf+4DFxv4vDrAo6tL+ayikZG9Erl2Ui6DMjwHnd/tdJCT5CCnjauDGWNojNj4grEhB3vDcmxrUx+KEoraTO6fykm5Sbo6KSLSRMFYRORrMMbApnXYSxfDho/AnYA1dTpW/nysvgMO+dqGcJR/fFLBK1uqSUlwctPUvkwfnNZuAdWyLJLcTpLcTnrTfkvtiogc7RSMRUS+AhMOYT5Yjln6EhTthvRMrHO/gXX6HKzUtEO/1hhW7q7nD2vLqG6MMGtoBpeN6dXq+FkREelcCsYiIi0wtg0VpVBcgCnaDUW7Y9viwthwif6Dsb51E9bE07Dch78qW1wf4rE1pXxc7Gdwpocfn9aP43ISO+GTiIhIWykYi0iPZmwbKsug6EsBuKQgdoPcXpk50HcA1uknYo2e0OaV5hrCUV7aXM3zGytxOSyuGd+bucMzO2ypYBER+foUjEWkRzI1lZjXnsesfAuCjfueyMiOzSZx2lmxbd7AWCBOSm7beY1hV3WQj4r9fFzkY3N5I1EDpw5K5arxfcjSkr0iIl2W/oYWkR7F1FZjXn8e8/YSMHZsKrWhI2MBOG8AVlLKVz5nXSDCupIGPi728XGRn+pAFIDBmR7OHZHFlAGpGjYhItINKBiLSI9g6mowS/5JzYq32Z7Ylx1Tv8X2PsdTEnKQGnCSUeQko8pHRmKATK+LDK+TjMSmrdeFx7VvBbiobdhWGeCjYh8fFfnZXhnAAKkJDsb0TWZcXgpj+ibr6rCISDejv7VF5KjlD0XZXljFttUfs62snh3JIymfdDIAFtAv5KB/egL+kE1hXYgNpQ3Uh+wWz5XkdpDhdZHudVJQG8QXsnFYMCw7kYtH5zCubzJDsrwaOywi0o0pGIvIUcEYw46qIG8W7GH97kq2lfsp8kebnh1In0w/x+Wmc3a/TIZlJ3Jslock98HTpIWjhtpghJrGKDWBCDWBCNWNEWoCex9Hmdw/lfF5yYzOTdZUayIiRxEFYxHptsJRm09LG1hV6GN1oY+qxggA2VaIIZU7OKP2C4b1zWDIzBmkDzy+Ted0Oy1yktxtXmVORESOHgrGItKt+IJRPiyKBeGPivw0Rmw8TouxOW4mRncxZtULZNaWYo0/Getbl2D1GxjvkkVEpJtQMBaRLq/MF2ZVYT2rC31sLGsgaiDD6+SUQalMsqoY9fGreJatBGPwTDqN8Fm3Y/UfHO+yRUSkm1EwFpEuqdQX4q2dtawq9LGrOghA/7QEzh2RxaTeCQzb9gHWkldhzxeQlII181ys0+eQMWIUFRUVca5eRES6IwVjEekyIrZhTaGPJdtrWF/sB+D4XolcMbYXk/unktdYgVn+EubJN6HRDwMGY11xY2xZZo8nztWLiEh3p2AsInFX6gvxxvZa3txRQ3UgSnaii0UnZpM/JIOcRAd8+hH2k69ib/gInM7Y+OHpZ8OQ49u0LLOIiEhbKBiLSFxEbMOaPT7e2FbDx8V+LAvG9U1m9rAMxuel4KipwLz3MvbbS6C8BNKzsOZfinXqLKyMrHiXLyIiRyEFYxHpVGW+MP/ZUcN/dtRS3RghO9HFRSdmM3NIBjm+csxHb2D+/D72rq2xFwwbiXXe5Vhjp2C59FeWiIh0HP0rIyKdYktFI898WsFHRbGxw+Pzkpk1sQ/jKcfx8X8wL7yPveeLWONBQ7HOvxxr7FSs3H5xrFpERHoSBWMR6XBv7azlkVUlpCY4WDgqm5meKnI2vIlZ+j6UFWMsC4aOwFp0TSwMZ/eKd8kiItIDKRiLSIexjeHv6yt4bmMlJ2Y6ubXmXVL+9h5UV2CcTjhuNNas87DGTsZKy4x3uSIi0sMpGItIhwhGbH79fjErdteTnx7k26/fhzsahhPGYi34JtZJk7CSU+JdpoiISDMFYxFpd9WNEe59u5DtlQGu8Oxh/uJfYx0zDMf1t2NlZse7PBERkRYpGItIu/q8OsA9ywupC0b5of99Ji9/EWvqDKzLrsdyJ8S7PBERkVYpGItIu/lwj49fvldEktNwz86nGfLFx1iLrsY6c74W4hARkS6vTcF43bp1PPnkk9i2zZlnnsmCBQsOeL6iooJHHnkEv9+PbdtceumljBs3rkMKFpGuxxjDK1uq+eNHZRzjtblt5a/JjvhwfP8nWCNOind5IiIibXLYYGzbNk888QR33HEH2dnZ3HbbbUyYMIH+/fs3t/nnP//J1KlTmTVrFoWFhfz85z9XMBbpIaK24fEPS3l9Ww2TE+q56Y1f4M3NxXHDA1i9cuNdnoiISJsdNhhv376d3Nxc+vTpA8C0adNYs2bNAcHYsiwaGhoAaGhoIDNT0y6J9AT+UJRfvlfEx8V+zovs5BvLH8MxbiqOb92E5U2Md3kiIiJfyWGDcVVVFdnZ++4iz87OZtu2bQe0WbhwIffccw9LliwhGAxy5513tn+lItKllPpC/M/yQorqQlxf/jb5G1/FOvdSrLkXYTkc8S5PRETkKztsMDbGHHTsyzfRrFixgjPOOIN58+axdetWfvOb3/DAAw/g+NI/jkuXLmXp0qUA3HfffeTk5BxJ7V+by+WK23tL51N/t78tZT5++MYOouEId2/7O6MqtpL24/vwTj4t3qWpv3sY9XfPov7ueTq7zw8bjLOzs6msrGx+XFlZedBQibfeeovbb78dgOHDhxMOh6mvryc9Pf2Advn5+eTn5zc/rqioOKLiv66cnJy4vbd0PvV3+9pY2sA9bxeSbIf4n1UP0y/JwvrxL/D1G4ivC/w+q797FvV3z6L+7nnaq8/z8vLa1O6wP+8cMmQIxcXFlJWVEYlEWLlyJRMmTDigTU5ODhs2bACgsLCQcDhMWlra1yhbRLqyD/f4uHtZAZnRBu597376DeyL4/YHsPoNjHdpIiIiR+ywV4ydTidXXXUV9957L7ZtM336dAYMGMAzzzzDkCFDmDBhApdffjmPPfYYr776KgDXX3+95iwVOcq883kdD60s4hjLz53v/pL0cROxvvV9LKcz3qWJiIi0C8u0NIi4kxQVFcXlffWjmJ5F/X3kXt9azWNrShlp1XLbO78iedrpWN+8DsvR9UKx+rtnUX/3LOrvnqezh1Jo5TsRaZUxhuc2VvK39RVMpIJb3n4Q7xlnYS26RjNPiIjIUUfBWERaZIzhqY/L+dfmKk63i7jh3Ydxz1qAdcEVGiolIiJHJQVjETlI1Db8dnUJS3fUMje8i6tW/A7nvIux5l2sUCwiIkctBWMROUA4avPAiiLeL/BxUWAziz54EscFV+A464J4lyYiItKhFIxFpFlj2Obn7xSyvqSBb/k+Yt6HT2Nd8h0cM86Jd2kiIiIdTsFYRACoD0b56bICtlcG+F7NCmasfwnr8u/hOHVWvEsTERHpFArGIkJ1Y4S73ixgT32QWyuWMnnTUqyrbsYx5Yx4lyYiItJpFIxFejh/KMrdbxVQ6gtxx55XGL19BY7v/BBr/LR4lyYiItKpFIxFerBw1HDfO3soqA3y/wpfYvSuD3BcdzvWSRPjXZqIiEinUzAW6aFsY3j4g2I+KW3gxuL/MOaL1ThuvBNr5Nh4lyYiIhIXCsYiPdRf1pXzzud1fKPyA6bvfDsWikecFO+yRERE4kZruor0QC9/VsULm6qYXbeR8ze/hOP62xWKRUSkx1MwFulhVu6u44m1ZUxq+Jxr1v8d53W3YY0aF++yRERE4k7BWKQH2VjWwIMrihgeLOXmj5/Afe2tWKN1o52IiAgoGIv0GLtrg9y7vJDewVpuX/sYiVd/H2vMlHiXJSIi0mUoGIv0AJUNYX7y5m7cAT93rv0daZd/F2v8yfEuS0REpEtRMBY5yvlDUX7yVgE+f4A71v2ePpdchmPy6fEuS0REpMtRMBY5ioWjNj9/u4DCmgA//PQphpx/Po5pM+JdloiISJekYCxylLKN4dcri/i0LMANnz3L2LNn4jhlZrzLEhER6bIUjEWOUn/6qJR3d/v45s7XmD59Ao7Tz4p3SSIiIl2aVr4TOco0hKO8tLmSf31Ww1l7VnL+pGNxnHlOvMsSERHp8hSMRY4CEduwrtjP27vq+KCgjpANU8o/4ZpR6ThnL4h3eSIiIt2CgrFIN2WMYUtFgLc/r+W9L+qpC0ZJsSJML/6Y00rXMuKMU3HOvSDeZYqIiHQbCsYi3UxhbZC3P6/jnc/rKPGFSXBaTPT4OG3n64wpWIt77CQct9yKldsv3qWKiIh0KwrGIt1AVWOEdz+v4+3P69hRFcBhwYl9krgo08ek5X8iqWA7HHscjh/eizV0ZLzLFRER6ZYUjEW6uLd21vKbD4qxDQzJ8nDVuN6c4q4m46XHYfN66JWL47s/hPEnY1lWvMsVERHpthSMRbqw1YX1/OaDYkb1TuI7E/vQP1qP+defMR8sg6QUrEXXYJ0xB8vljnepIiIi3Z6CsUgXtbG0gV++V8SQLC+3Tc7C+59nsJe+BMZgzVqANXchVlJKvMsUERE5aigYi3RBu6oD3PN2Ib2T3dzZpxLPXXdi6muxJp+Odd5lWNm9412iiIjIUUfBWKSLKa4PcfdbBSS6Hfx/ni2kPPoY9BuE46a7sAYNjXd5IiIiRy0FY5EupKoxwl1vFRC1DT9tXEHOay/AmCk4rr4Zy5sY7/JERESOagrGIl2ELxTlJ28VUNsY4SflS+j/yZtYs8/HOv9yLIcj3uWJiIgc9RSMRbqAYMTm3uWFFNYG+X9fvMiwL9ZgXflfOE7Oj3dpIiIiPYaCsUicRWzDL9/bw+byBv57xz85qWozjpt/inXcqHiXJiIi0qMoGIvEkW0Mv/mgmDV7/Hx322KmRffguP2XWL3z4l2aiIhIj6NgLBInxhieXFvK8l11XLzr38xO8+P47i+xkjU3sYiISDwoGIvEyfOflPLSlhrmFr7HRYPcOC65C8ulP5IiIiLxon+FReLg3+sL+esGH6eWfszV43rjyJ+HZVnxLktERKRHUzAW6USBiM2bq7fyh50242q2818zhuIaMzHeZYmIiAgKxiIdLmob1pf4Wf5ZKauKGglYLkb69/DDeSeSMGhwvMsTERGRJgrGIh3AGMO2ygBvf17Hu7tqqQ3ZJEcaObViA6cPTGbkN2fiTE6Nd5kiIiKyHwVjkXZUXB/i7V11vP15LUX1YdzYjK/6jNOLPmT88XkkXHUJVmZ2vMsUERGRFigYixyhmkCE976o4+1ddWytDGABJ3gCnPfFW0wp+IDkE0bjuPFarLyB8S5VREREDkHBWOQrqmgIs6mskU1lDWwub+SLmiAGOCbDwxV9Gjl55T/IKfgMhhyP45a7sYaNjHfJIiIi0gYKxiKHYBtDYW2ITeUNbC5rZFN5A2X+CABel4PjeyUybWAqk00ZA17/HWzfDLn9cFx/O4yZrCnYREREuhEFY5H9RO3YTXObyhrYVN7IZ+UN1IdsADK8Tkb2TmL+8YmM7J3EMRkeHOXF2M//HtZ9AOlZWJddj3XyTCynM86fRERERL4qBWORJhHb8LO3C1lb5AcgLzWByQNSGdkrFoRzU9zNV4BNgx/zz6ew33wZXG6sBd/Eyp+P5fHG8yOIiIjIEVAwFiE2vdpvV5WwtsjP5WN6ceaQdDK8B//xMHYUs+JNzIt/AV8d1rQzsc67DCs9Mw5Vi4iISHtSMBYB/v5JBW/urOXiE7O54ISWp1MzWzdgP/04FOyCoSNw3HQX1qChnVypiIiIdBQFY+nxlmyr5tkNleQPSefiE3MOet5UlGKefwqzdgVk5WB951asCafoxjoREZGjjIKx9GirCut5bE0p4/OSuX5S7gFh1wQDmNefx/z7RXBYWPMuwZp9PpbHE8eKRUREpKMoGEuPtaWikV+9V8SQLC8/PLUfTkfTjXW2jVn9Nuaff4KaKqxJp2FdcAVWVq84VywiIiIdScFYeqQ9dSH+Z3khWYku7jijP16XAwBTuAv7z4/Arq0waCiO7/4Ia+iIOFcrIiIinUHBWHqc6sYIP1lWgAO4e8aA5tknzOb12L/9GbgTsK68CWvqdCyHI77FioiISKdRMJYepSEc5X+WF1DTGOHemQPpm5oAgL3mXcwT/wt98nDcdDdW1sE34YmIiMjRTcFYeoyIbbj/3SJ2VQf5f6f3Z1h2IgD2m69gnnkchozA8b07sJJT4lypiIiIxIOCsfQIxhj+74Ni1hX7uXFKLhP6pWCMwbz4F8zrz8OYyTi+/QOsBM04ISIi0lMpGEuP8Nf1FSzbVcclo3PIH5KBiUYxf3kEs2Ip1mmzsS69FsvpjHeZIiIiEkcKxnLUe/GTYp7fWMmsoeksGpWNCQaxf/8L+GQN1jkXY82/RIt1iIiIiIKxHN0+KKjnwXf3MLFfMtdOzAV/Pfb/3QM7t2B941ocZ8yNd4kiIiLSRSgYy1HJNoaXP6vmz+vKOL5PKj84pS+Omgrsh+6G8uLY/MTjp8W7TBEREelCFIzlqFPTGOGh94v5uNjP5P4p3H32SIJbPsV+6C4INuL4/k+xjhsV7zJFRESki1EwlqPKR0U+Hnq/mMawzbUT+3DWsAy8uzbT+IsfgduD49afYw0YHO8yRUREpAtSMJajQjhq8+d15bz0WTWDMjzcc2YeAzM8mHWrqH78l5DZC8f378bK6RPvUkVERKSLUjCWbq+wNsivVsQW7jh7eAZXjO2Nx+XAFOzC/t39uI4djn3dbVip6fEuVURERLowBWPptowxLN1Ry+MflpLgcnD76f2Y3D819lwkgv3kQ5CcQuYdD1AVCse5WhEREenqFIylW/KFovx2VQkrdtczuk8S35/Wl+wkd/Pz5rXnoGAXjutvx5GWDhUVcaxWREREugMFY+l2NpU18OCKIqoaI1w2phfnjcjC6di3QIcp2IV57VmsSadjjZ0Sx0pFRESkO1Ewlm4jahue21DJMxsq6J3s5r5Zgxiek3hAm31DKFKxLvl2nCoVERGR7kjBWLqNJz8q4+Ut1ZxxTBrfndSHJLfzoDbNQyhuuB0rJS0OVYqIiEh3pWAs3cKHe3y8vKWas4dn8J2JuS22Mbt3xoZQTD4da4yGUIiIiMhX44h3ASKHU9UY4dfvF3NMhocrx/VusY2JhLGf/HXTEIrvdHKFIiIicjRQMJYuzTaGh1YWEYjY/OCUPBKcLX/Lmteeg8JdOC67His5tZOrFBERkaOBgrF0af/aVMX6kga+PaEPA9I9LbaJDaF4TkMoRERE5IgoGEuXtbWikb+uL2fawFRmDml51brmIRQpaRpCISIiIkdEwVi6pIZwlAdWFJGV6OKGSblYltViu+YhFN/UEAoRERE5MgrG0iU9tqaUMn+YW07OI8Vz8LRssN8QiilnYI2Z3MkVioiIyNFGwVi6nOW7alm+q45Fo3IY2TupxTaxIRQPxYZQXKyFPEREROTIKRhLl1JcH+LR1aWM7JXIwlHZrbYzrz4HhZ/juOwGDaEQERGRdqFgLF1GOGp4YEURTgfccnIeTkcr44p378C83jSE4qRJnVyliIiIHK0UjKXL+Psn5WyrDPC9ybn0Sna32OaAWSg0hEJERETakYKxdAnriv28sKmK2UMzmDYwrdV2GkIhIiIiHUXBWOKuNhDhoZVF9E9L4OrxLS/5DPsPoZiuIRQiIiLS7hSMJa6MMTz8fjG+kM2tp+ThcbWy5HM4hP2HByElXUMoREREpEMoGEtcvbKlmg+L/HxrXG+OyfS22s68+BcoLsBx5X9hJad0YoUiIiLSUygYS9zsqg7w1MflTOyXwtzhGa22M599gvnPYqwz5mKNGteJFYqIiEhPomAscRGO2jywoog0j5P/mnKIJZ8b/LGFPPr0w7rwys4tUkRERHoUBWOJi2c+raSgNsSNU3JJ87pabWf+8XuoqcJx9c1YntaHWoiIiIgcKQVj6XQ7qwL8c1MlM45NZ1xe6+OFzdqVmA+WYZ19Edbg4Z1YoYiIiPRECsbSqSK24eEPikn3OLl63CGmZqupwv7rIzBoKNbcizqxQhEREempFIylU72wqZJd1UGunZRLisfZYhtjDPaffgPBII6rb8FytT7UQkRERKS9KBhLp9ldG+SZTys5eWAqUwa0vmqdeeffsGEt1oVXYvXt34kVioiISE+mYCydImob/u+DYhLdDr4zsU+r7UxpEebZJ2DkGKwz5nZihSIiItLTKRhLp3hlSzVbKgJ8e3xvMlqZhcJEo9h//F9wuXFceROWQ9+eIiIi0nmUPKTDFdeH+Ov6cib2S+a0Y9JabWdefx52bsH6xrVYmdmdWKGIiIiIgrF0MNsYHllVgsthcd2kQyzk8cV2zCtPY008Fcek0zq5ShEREREFY+lgb2yv4dPSBr41rjfZSe4W25hQEPsPD0JqBtY3ru3kCkVERERiFIylw5T7wzz1UTmjc5OYOSS91XbmhT9DSSGOb/0XVnLrs1WIiIiIdCQFY+kQxhgeXV2CbQzfm3yIIRSb12PefBlrxjlYI8d2cpUiIiIi+ygYS4dYvquOtUV+LhvTiz4pCS22MQ0+7Cd/Dbn9sM6/opMrFBERETlQm5YUW7duHU8++SS2bXPmmWeyYMGCA55/6qmn2LhxIwChUIja2lqeeuqpdi9WuoeaxghPrC3l+JxEzj4us9V25h+PQ20Vjh//Esvj6cQKRURERA522GBs2zZPPPEEd9xxB9nZ2dx2221MmDCB/v33rUh25ZVXNu+//vrr7Nq1q0OKle7hsQ9LCUQMN07JxdHaEIodn2E+WIY19yKswcM6uUIRERGRgx12KMX27dvJzc2lT58+uFwupk2bxpo1a1ptv2LFCk455ZR2LVK6j5W761i5u56LR+fQP73lq8DGGOzn/gjpmVhzLujkCkVERERadthgXFVVRXb2vsUWsrOzqaqqarFteXk5ZWVljBo1qv0qlG6jLhjlsTWlDMnyct6IrNYbfvQ+7PgM69xvYHkTO69AERERkUM47FAKY8xBx1qbYWDFihVMmTIFRytL+S5dupSlS5cCcN9995GTk/NVam03Lpcrbu99NHv031uoD9k8dP7x9OmV0mIbEw5T+a+/4Bp4LFnzF2E5nR1el/q7Z1F/9yzq755F/d3zdHafHzYYZ2dnU1lZ2fy4srKSzMyWb6hauXIlV199davnys/PJz8/v/lxRUXFV6m13eTk5MTtvY9W64r9LPmsnItGZZNpBaioCLTYzl76EqZkD46b7qKyurpTalN/9yzq755F/d2zqL97nvbq87y8vDa1O+xQiiFDhlBcXExZWRmRSISVK1cyYcKEg9oVFRXh9/sZPnz4V69WurWIbXj8w1L6prq5aFR2q+2M34d55RkYORZr1PhOrFBERETk8A57xdjpdHLVVVdx7733Yts206dPZ8CAATzzzDMMGTKkOSS/9957TJs2rdVhFnL0em1rNYV1Ie44vT9uZ+v/1zKvPQcNPhwLr+y84kRERETaqE3zGI8bN45x48YdcGzRokUHPL7ooovaryrpNmoCEf7xSQXj+iYzoV9yq+1MeQnmrZexpp2J1X9wJ1YoIiIi0jZa+U6OyF/XlROM2Fw9ofchf1pgXvwLOJxY536jE6sTERERaTsFY/natlcGWLqjlnnHZ9E/rfWV68zOLZg172LNOg8rs/UxyCIiIiLxpGAsX4sxsRvu0rzOQ99wt3cxj7QMrNnndWKFIiIiIl+NgrF8LW9/XsdnFY1cPqYXyQmHmIv44/dh+2Yt5iEiIiJdnoKxfGWNYZs/fVzOsGwvM45Nb7WdiYSx//knyBuIdXJ+q+1EREREugIFY/nKnt9YSVVjhG9P6IPjUDfcvb0EyopxXPitTlnhTkRERORIKBjLV1JcH+Jfm6uYPjiN43JaHxphGnyYl5+GESfBqHGtthMRERHpKhSM5Sv540dluBwWl4/tfch2zYt5XPgtLfoiIiIi3YKCsbTZR0U+Vhf6uGhUNlmJra8NYypKMW++jDV1BtbAYzuxQhEREZGvT8FY2iRiG55YW0bfVDfzj888ZNvYYh4OLeYhIiIi3YqCsbTJq1uqKawLcfW4PridrX/bmF1bMavfwZq5ACsrpxMrFBERETkyCsZyWDWBCE9/WsG4vslM6JfcarvmxTxS07HOOr8TKxQRERE5cgrGclh/WVdOMGJz9YTeh76Rbt0q2LapaTGPpM4rUERERKQdKBjLIW2vDPDmjlrmHZ9F/zRPq+1MJIL9/FPQdwDWKTM7r0ARERGRdqJgLK0yxvD7D0tJ8zq5aFT2oduuXAplRTguvFKLeYiIiEi39P+3d/fBUVd5vsc/p7vzHAhJR0EkKASf8AkxDIgoxERBZ3Qcd9QarzvuunXLqpmdmXX/2NUZV91y3bJ2x3Ju1XVr5k5x0drau4XOg08oOlFgRgENIgooAZTxMRA7CUmgf52k8zv3jyaBkIduIPl1kvN+/dPp5BfzHY4VP3zne84hGGNIG/7croaYp+/PO01FuUOHXZtMyr78G2nWudLFVQFWCAAAMHIIxhhUvLtHT733tc6J5uua2SXDPmvf2SA1Nyn0zdu5zAMAAIxbBGMM6nc7W9TqJfU/q6YqNEzYtX5PqltcMUu6hG4xAAAYvwjGGCDpW72296AWVRTrvPKCYZ+19W9KB76kWwwAAMY9gjEGeO+rw2rr7FFNuhEK35d9+Vlp+kzpskUBVQcAADA6CMYYYN2+Nk3OC2v+9OLhH9y2WfrqM5kbbpUJ8a8SAAAY30gz6OdQV4/e+eKQrjp7siKhYWaLrZW/5hnp9OkyC5YEWCEAAMDoIBijn42fdajbt6qeNXn4Bz/YIn32yZFuMecWAwCA8Y9gjH7WfdKmGZNzNacsf8hnUt3i1VL0dJmFSwOsDgAAYPQQjNHnwKEuffi1p2WzJg9/wsSH26R9u2Vu+K5MJBJcgQAAAKOIYIw+6/e1S5KWzRr+NIKh8kgAACAASURBVAp/zWqptFzmipogygIAAAgEwRiSUuMR6/a16aKphTqtKGfo5xp2SHs+lFl+i0zO0M8BAACMNwRjSJJ2NyfU2NGddtOdv2a1NHmKzFXXBlQZAABAMAjGkJTadJcbNlo8c9KQz9iPd0kfvS+z/DsyuXkBVgcAADD6CMZQd4+vNz9t16IZk1SYM/TRa/5Lq6XiSTJLrw+wOgAAgGAQjKEtXx1WR5evZcOMUdhP90o73pWp/bZM3tBHuQEAAIxXBGNo/b42TckPa94ZRUM+47/0jFRYJHPNtwKsDAAAIDgEY8e1d/Zoy5eHdPXZkxUe4gpo+8U+adtmmZobZQoKA64QAAAgGARjx731abuSvlQ9zNnFds2zUn6BTM1NAVYGAAAQLIKx49bta9NZJXmaVTr4KRO28XPZd9+Sqf6mTFFxwNUBAAAEh2DssK/au9QQS2jZ7KGvgLYvPyvl5Mpc++2AqwMAAAgWwdhh6/a1yUhaevbgp1HYpkbZt/8os3SFzKThr4kGAAAY7wjGjvKt1fp97bp0WqGihYNf7Wxf+Y0UDstc952AqwMAAAgewdhRH33tqelwt5YNsenONjfJbnpD5qrrZKaUBVwdAABA8AjGjlq/r035EaNFFYNfAW1f/Z0kI7PilmALAwAAyBKCsYO6eny99WmHFlVMUkHOwH8FbPyw7MY3ZBYulSk7LQsVAgAABI9g7KD6Lw7pcLc/5NnFdtMbUmdC5ppvBlwZAABA9hCMHbRuX5vKCiK6eOrAW+ys78uue1mafZ7MWXOyUB0AAEB2EIwdczCR1NavDmvZrCGugN71vnTgS5nqG4IvDgAAIIsIxo7505/b1WM15GkU/htrpEklMpcvCbgyAACA7CIYO2b9vnbNLs3TWVMGXgFtYwekD7akjmjLGfxsYwAAgImKYOyQz9s6tbclMfTZxRvWSpLM1SuCLAsAAGBMIBg7ZP2+doWMdPUgV0Db7i7ZN1+T5n1DJsoRbQAAwD0EY0ekroBu02VnFKm0IDLg67b+T9KhDoWqOaINAAC4iWDsiB0H4orFk0OPUax7WZo2Qzr/koArAwAAGBsIxo5Yt69dBZGQFs4oHvA1u2+39Oc9MtU3yJhBjnADAABwAMHYAfHuHm38rENXnjVJeZFBroBet0bKK5C54posVAcAADA2EIwd8Mz2ZiWSvlacM2XA12xHm2z9mzJXVMsUDLwJDwAAwBUE4wnus7ZOvbCrRbWVJTonWjDg6/bNP0jJbm66AwAAziMYT2DWWv26/oDyc0L6/ryBR7BZv0d2/SvSeRfLTJ+ZhQoBAADGDoLxBPbmpx364EBcd156mkryBx7Rpg/qpZavOaINAABABOMJy+v2tWprk2aX5mn5nIGzxZLkr3tZKi2X5i0MuDoAAICxh2A8QT2zI6ZmL6l7FkxTODTwCDa7/wvpw20yVy+XCYezUCEAAMDYQjCegD5v69TzH6U23J1/2sANd5JSs8XhiMzV1wVcHQAAwNhEMJ5grLX6P1uG3nAnSTbhyW58XebyK2UmlwZcIQAAwNhEMJ5g3vqsQx/sj+t/XDLEhjtJdvN6yYvLXMOmOwAAgF4E4wnE6/b1f99Nbbgb7DIPKdVRtutflmbOlmafF3CFAAAAYxfBeAJJt+FOkrR7p/TlpzLV35QxQzwDAADgIILxBNG74a5m9tAb7iTJrlsjFRbLLLg6wOoAAADGPoLxBNBvw91lg2+4kyTb2iz73iaZJdfK5OUFWCEAAMDYRzCeADYes+FuyhAb7iTJ/vFVyVqZZdcHWB0AAMD4QDAe57xuXyu3NmnWMBvuJMkmu2X/uFa66HKZ06YFWCEAAMD4QDAe557ZEVNzPKl7FkwdesOdJLt1k9R+UKFqjmgDAAAYDMF4HPuirVMv7GrRNbNLdMFphcM+a9etkU6bJl14WUDVAQAAjC8E43Gqd8NdXjiku4bZcCdJ9tOPpb0fySy7XibEkgMAAAyGlDRObfy8Q+/vj+t/XDr8hjtJsq/9XsovkFlyXUDVAQAAjD8E43HI6/a18t30G+4kyTY3yW55U+bq5TKFRQFVCAAAMP4QjMeh333YnNpwVzX8hjtJsnUvSMbI1NwYUHUAAADjE8F4nDnU1aOXGlp1RcUkXXB6mg13hw/J/uk1mQVXyZQNP4cMAADgOoLxOPNSQ6vi3b5uuyia9ln7x7VSZ0Lmuu8EUBkAAMD4RjAeR+LdPXpxV4u+MaNYs8vyh33WdnfLvv6iNHeeTMWsgCoEAAAYvwjG48jLuw/qUFeG3eJ3NkhtrQotp1sMAACQCYLxOJFI+nr+oxbNP6NI50QLhn3W+r7sq7+XZsySLpgXUIUAAADjG8F4nHh1z0G1d/botovTd4u1c6vU+LnM8ptlzPCnVgAAACCFYDwOdCZ9/f7DZl0ytTDt1c+S5L/6e6m0XKbqqgCqAwAAmBgIxuNA3cdtak1k1i22f94jNWyXqb1RJjL8jXgAAAA4imA8xnX3+Prth82ae1qBLkpzbrEk2deekwoKZa5aHkB1AAAAEwfBeIx745N2NceTuu3i8rTzwjZ2QHbLW6nrnwvSh2gAAAAcRTAew5K+1W92NuucaL7mTcugW1z3ghQyMjU3BVAdAADAxEIwHsPW72tT0+Fu3X5RBt3iwx2yb/5B5htXy5RmcHIFAAAA+iEYj1E9R7rFs0vzVHVmUdrn7fpXuP4ZAADgFBCMx6g/fdquxo5u3ZZJt7i7W/aNl6QLL5OZcXYwBQIAAEwwBOMxyLdWz+5o1lkleVpYUZz2ebt5ndR+UKHltwRQHQAAwMREMB6DNn3WoS/au/Tdi6IKpesW+37qiLaKWdL5lwRUIQAAwMRDMB5jfGv1zI5mnTk5V1fOnJT+G7a/K+3/Qmb5LVz/DAAAcAoIxmNM/ReH9OeDnbr1wqjCofRB13/td1JZuczlVwZQHQAAwMRFMB5DrLVavaNZ04pzdPXZk9M/v2+3tHunTO23uf4ZAADgFBGMx5CtXx3Wxy0JfTfDbrF99fdSQZHMVdcGUB0AAMDERjAeI3q7xacVRrRsVkn657/eL7t1k8zSFTL5XP8MAABwqgjGY8QHB+JqiHn6iwujygln0C3+w/NSKCRT860AqgMAAJj4CMZjxDPbYyoriKimMoNu8aF22bfqZBYulZnC9c8AAAAjgWA8BuxsimtHk6db5pYpN5x+SeyGtVJXJ9c/AwAAjCCC8RjwzPaYSvLDum7OlLTP2mS37Lo1qeufz5wZQHUAAABuIBhnWUPM07b9cd18QZnyIhl0i7e8KbW1KlT77QCqAwAAcAfBOMue2R7TpLywrj+nNO2z1lrZP7wgnVEhXXhZANUBAAC4I6NbIbZt26ZVq1bJ933V1NTo5ptvHvDMxo0b9eyzz8oYo7POOks/+clPRrzYiWZvc0JbvjqsOy8tV0FOBn9H2fuR9NnHMnf+gOufAQAARljaYOz7vlauXKkHHnhA0WhU999/v6qqqjRjxoy+ZxobG/Xcc8/pkUceUXFxsdra2ka16InimR0xFeWG9M3z0neLJcmve0EqmiSzqHqUKwMAAHBP2jbl3r17NW3aNE2dOlWRSESLFy9WfX19v2def/11LV++XMXFxZKkkpL0R4657s+tCb39xSHddF6ZCnPCaZ+3X++X3tssc/Vymby8ACoEAABwS9qOcUtLi6LRo2flRqNR7dmzp98zX331lSTpn/7pn+T7vm699VbNmzdvwD+rrq5OdXV1kqTHHntM5eXlp1T8yYpEIln72b3+1zu7VJgb1vcXz9Hk/PQTLR0v/j/FQ0bRW+5UOMu1jzdjYb0RHNbbLay3W1hv9wS95mkTmbV2wOeOn2/1fV+NjY166KGH1NLSogcffFCPP/64ioqK+j1XW1ur2travvexWOxk6z4l5eXlWfvZkvRZW6fW7YnpLy6MquvQQcUODf+8TcTl/+EFmcuvVKtCUhZrH4+yvd4IFuvtFtbbLay3e0ZqzadPn57Rc2lHKaLRqJqbm/veNzc3q7S0/0xsWVmZFixYoEgkotNPP13Tp09XY2PjCZbsjmd3NCsvYvTt8zObLbZvvSF5cZnam0a5MgAAAHelDcaVlZVqbGxUU1OTksmkNm7cqKqqqn7PfOMb39COHTskSe3t7WpsbNTUqVNHp+Jx7sv2Lr35abuuP6c0oxEK6/uyr78gVZ4vM+vcACoEAABwU9pkFg6Hdffdd+vRRx+V7/uqrq5WRUWFVq9ercrKSlVVVenSSy/V+++/r3vvvVehUEh33nmnJk2aFET9485vdsYUCRndfEFZZt/wQb309X6Fbvn+6BYGAADguIzOMZ4/f77mz5/f73O3335738fGGN1111266667Rra6CWZ/R5fW72vXN88r1ZSCjP7oU0e0lZVLl10xytUBAAC4jZvvAvTbD5sVNkbfybBbbD/fJzVsl7nmWzLh9Ee6AQAA4OQRjAPy9eFuvfFJm66dU6JoYU5G32Nff0HKzZNZct0oVwcAAACCcUB+uzN1ssctc6Npnkyx7Qdl394gs7hGpqh4NEsDAACACMaBaI536w8ft+ma2SU6rSjDbvGGtVIyKVNz4yhXBwAAAIlgHIjff9gi31p998IMu8Xd3bLrX5YurpKZduYoVwcAAACJYDzqWr2kXt17UMtmlWhqcW5G32Pr/yi1H1SICz0AAAACQzAeZc991KKkb3Vrpt1ia2XrXpCmz5QuuHSUqwMAAEAvgvEoak8ktXZPq646a7KmT86sW6zdO6XP98nU3iRjzOgWCAAAgD4E41H0/K5WdSatbr0os26xdORCj+LJMguXjmJlAAAAOB7BeJR0dPZoTUOrFs+cpIqSvIy+x369X3r/bZmlK2RyM/seAAAAjAyC8Sh5qaFFXtLXbSfQLbavvyiFwjLLrh/FygAAADAYgvEoONzVoxcbWrVwRrHOLs3P6HusF5d9q05mwRKZKZmHaQAAAIwMgvEoWLO7VYe7fN1+cXnG32Pf+oOU8GQ4og0AACArCMYjLN7doxc+alHV9CJVlmXYLfZ7ZF9/SZozV+asOaNcIQAAAAZDMB5hb37aoY4uX7delHm3WO/XS7EDXOgBAACQRQTjEfbR155K8sI6rzyzbrF05Ii26OnSZQtHsTIAAAAMh2A8whpins4tL8j4cg67/wtp9w6ZZdfLhMKjXB0AAACGQjAeQR2dPfqyvUvnlxdk/D1203rJhGQWVY9eYQAAAEiLYDyC9jR7kqRzMxyjsL4vu3mddOE8mSllo1kaAAAA0iAYj6CGmKeQkeZEM5wv3r1Davla5oprRrcwAAAApEUwHkG7YgnNLMlTYU5ms8J20zqpoFBmHpvuAAAAso1gPEJ8a7Un5um8DOeLbWdC9t2NMpdfKZObN8rVAQAAIB2C8Qj5sr1Lh7v9jI9ps+9tkjo9mSvYdAcAADAWEIxHSEMstfEu447xpnVS+VRpztzRLAsAAAAZIhiPkIaYp6LckKZPzk37rG1tlj56X2ZRtUyIJQAAABgLSGUjpCGW0HnRAoUyuNjDbl4vWStzxbJRrwsAAACZIRiPgHh3jz472JnRGIW1VnbTG9KcC2ROnx5AdQAAAMgEwXgE7G1OyCrDiz0++1hq/JxNdwAAAGMMwXgE7Dqy8e7caAYd441vSJEcmaolo10WAAAATgDBeATsjnmaMTlXxXnDX+xhk92y7/xRZt5CmcLigKoDAABAJgjGp8ham9p4l8kxbTvelQ61M0YBAAAwBhGMT9H+Q91q7+zJKBj7m9ZJk0qkuZcFUBkAAABOBMH4FB292GP4jXf2cIf0fr3MwmUykUgQpQEAAOAEEIxPUUPMU34kpIqSvGGfs+/8SepJMkYBAAAwRhGMT1FDLKFzo/kKh4a/2MNuekM68yypYlZAlQEAAOBEEIxPQWfS159bEzo3zXyx3f+FtG+3zBXXyGRwMx4AAACCRzA+BXtbEuqxGcwXb1onmZDMwqUBVQYAAIATRTA+BUc33g3dMba+L7t5nXThPJkpZUGVBgAAgBNEMD4Fu2OephXnqCR/mFMmdu+QWmIyi9h0BwAAMJYRjE+StVa7MrjYw258QyoolLlsUUCVAQAA4GQQjE9SLJ5Uq5ccfoyiMyG7daPM5VfK5A5/nBsAAACyi2B8kjKaL35vk9SZ4OxiAACAcYBgfJIaYp5yw0Znlw7dCbYb35DKp0pz5gZYGQAAAE4GwfgkNcQSmlOWr8gQF3vYlpi06wOZRdUyIf6YAQAAxjoS20no7vH1ccvwF3vYtzdI1spcsSy4wgAAAHDSCMYn4ZPWTiV9q/OHCMbW2tQV0HMukDl9esDVAQAA4GQQjE/C7iMb784d6sa7T/dKjZ+z6Q4AAGAcIRifhIaYp/LCiKKFOYN+3W5aJ0VyZKqWBFwZAAAAThbB+CQ0xLwhj2mzyW7ZdzbIzFsoU1gccGUAAAA4WQTjE9TiJdV0eJiLPT7YIh3qYIwCAABgnCEYn6DdaS728N94SSotl+ZeFmRZAAAAOEUE4xPUEPMUCUmzywZe7GE/+0Rq2C5zzTdlIpEsVAcAAICTRTA+QQ0xT7NK85UbHvhHZ+uel3LzZK5anoXKAAAAcCoIxiegx7fa05wY9Pxie7BF9p0/yVxZI1PEpjsAAIDxhmB8Aj492KmuHjvojXd2/cuS3yNTc1MWKgMAAMCpIhifgIa+jXf9L/awXZ2yG9ZKlyyQmcpNdwAAAOMRwfgE7Ip5mpIf1ulF/S/2sJvXS4faFbr229kpDAAAAKeMYHwCdh+52MMY0/c5a61s3QtSxSzp3IuyWB0AAABOBcE4Q+2dPfqqo3vg+cU735MaP5ep/Xa/wAwAAIDxhWCcoaEu9vDrnpdKSmW+cVU2ygIAAMAIIRhnqCHmKWSkOdGjG+/sV59JO9+TWXaDTCRnmO8GAADAWEcwzlBDzNPZU/KUHzn6R2brXpBycmWWXp/FygAAADASCMYZ6PGtdscS/cYobEe77Ob1MouWyUyanMXqAAAAMBIIxhn4sr1LXtLvd7GH/eNaqbtLppYLPQAAACYCgnEGdh238c4mu2XXvSxdeJnM9JnZLA0AAAAjhGCcgYaYp0m5IU2flNpgZ+vflNpaFKrlQg8AAICJgmCcgd0xT+ceudgjdaHH89IZFdKFl2W7NAAAAIwQgnEah7t69Hlb19GNd7t3Sp99IlN7Exd6AAAATCAE4zT2NCdkdXS+2K97XiqeLLNoWVbrAgAAwMgiGKfREPNkJJ0TzZdt+kp6/x2ZpStkcvOyXRoAAABGUCTbBWRTd4+vQ12+DnX16FBnjzq6evred3T26FBXj9796rAqSnJVlBuW//pLUigss+yGbJcOAACAEeZUMN7T7Ol/b94vL7lPbYkuJZJ2yGeNpOLckIpyw1p+zhTZ+CHZt+pkvnGVzJSy4IoGAABAIJwKxgWRkKYW5yg6qVA5tlvFeWFNyg2rKDesSXlhFeeGNCk3rOLcsApzQwods7nOf/X3UmdChiPaAAAAJiSngvGMkjz9dOkMlZeXKxaLZfx9tqdH9o2XpPMulpk5exQrBAAAQLaw+S4DdusmqeVrhbj+GQAAYMIiGGfA1j0vnX6GdMmCbJcCAACAUUIwTsN+vEv6pEGm5kaZEH9cAAAAExVJLw37+otSYZHM4ppslwIAAIBRRDBOw372iXTBpTL5BdkuBQAAAKOIYJxOwpMpLM52FQAAABhlBON0EnGJbjEAAMCERzAehvV7pM4EwRgAAMABBOPhJBKp1/zC7NYBAACAUUcwHk4innqlYwwAADDhEYyHk/BSrwV0jAEAACY6gvFwvFTH2DBKAQAAMOERjIfT2zFmlAIAAGDCIxgPp2+UgmAMAAAw0RGMh2F7N9/lEYwBAAAmOoLxcNh8BwAA4AyC8XA8jmsDAABwBcF4OAlPiuTIRHKyXQkAAABGGcF4OIk4YxQAAACOIBgPx/MYowAAAHAEwXgYtpNgDAAA4AqC8XC8OMEYAADAEQTj4SQ8ieugAQAAnEAwHk4iLsPmOwAAACcQjIeTYMYYAADAFZFMHtq2bZtWrVol3/dVU1Ojm2++ud/X169fr//8z/9UWVmZJGnFihWqqakZ+WqDlogzSgEAAOCItMHY932tXLlSDzzwgKLRqO6//35VVVVpxowZ/Z5bvHix/uZv/mbUCg2a7emRurroGAMAADgi7SjF3r17NW3aNE2dOlWRSESLFy9WfX19ELVlV8JLvRYQjAEAAFyQtmPc0tKiaDTa9z4ajWrPnj0Dnnv77bf10Ucf6YwzztBdd92l8vLyka00aIl46pVRCgAAACekDcbW2gGfM8b0e3/55ZfryiuvVE5Ojl577TU9+eSTeuihhwZ8X11dnerq6iRJjz32WNbCcyQSSfuzk4fb1Cxp8mlTlT/eQ77jMllvTByst1tYb7ew3u4Jes3TBuNoNKrm5ua+983NzSotLe33zKRJk/o+rq2t1X/9138N+s+qra1VbW1t3/tYLHbCBY+E8vLytD/bNn4lSeroTupQlurEyMhkvTFxsN5uYb3dwnq7Z6TWfPr06Rk9l3bGuLKyUo2NjWpqalIymdTGjRtVVVXV75nW1ta+j7ds2TJgY9641DdKwYwxAACAC9J2jMPhsO6++249+uij8n1f1dXVqqio0OrVq1VZWamqqiq98sor2rJli8LhsIqLi/WDH/wgiNpHV9/mO2aMAQAAXJDROcbz58/X/Pnz+33u9ttv7/v4jjvu0B133DGylWWZ9egYAwAAuISb74bSeaRjzKkUAAAATiAYD8XrDcZ0jAEAAFxAMB5KwpNyc2XC4WxXAgAAgAAQjIeSiDNGAQAA4BCC8VC8OGMUAAAADiEYD8EmPDrGAAAADiEYDyUR5wxjAAAAhxCMh5LwGKUAAABwCMF4KAlPhmAMAADgDILxUNh8BwAA4BSC8VDYfAcAAOAUgvEgbLJbSnaz+Q4AAMAhBOPBJLgOGgAAwDUE48F48dQroxQAAADOIBgPpjPVMeZUCgAAAHcQjAfjHRmlKCAYAwAAuIJgPJjEkVGKPIIxAACAKwjGg7C9m+84lQIAAMAZBOPBsPkOAADAOQTjwXBcGwAAgHMIxoPpnTHOz89uHQAAAAgMwXgwnifl5cuEwtmuBAAAAAEhGA+m02O+GAAAwDEE48F4cc4wBgAAcAzBeBA24XGGMQAAgGMIxoNJxDnDGAAAwDEE48EkPI5qAwAAcAzBeDBeXIbNdwAAAE4hGA+GjjEAAIBzCMbHsdamgjGnUgAAADiFYHy8ZLfUk+QcYwAAAMcQjI+X8FKvjFIAAAA4hWB8PC+eeqVjDAAA4BSC8fGOdIwNHWMAAACnEIyPlzjSMeaCDwAAAKcQjI/n9c4YE4wBAABcQjA+ju3tGDNKAQAA4BSC8fF6T6XgHGMAAACnEIyPx3FtAAAATiIYHy8Rl4yRcvOzXQkAAAACRDA+nheX8vJlQvzRAAAAuIT0d7yEx4kUAAAADiIYH8cm4pxhDAAA4CCC8fESHhvvAAAAHEQwPh7BGAAAwEkE4+MRjAEAAJxEMD6eF5dh8x0AAIBzCMbHY/MdAACAkwjGx7DWMkoBAADgKILxsbq6JN/nHGMAAAAHEYyP1RlPvdIxBgAAcA7B+Fiel3otIBgDAAC4hmB8rESqY2zoGAMAADiHYHysxJGOMTPGAAAAziEYH8s7MmPMcW0AAADOIRgfw/Z2jPMYpQAAAHANwfhYCTrGAAAAriIYH6tvxpiOMQAAgGsIxsfyPMmEpNy8bFcCAACAgBGMj5WISwUFMsZkuxIAAAAEjGB8rITHGAUAAICjCMbHsIk4ZxgDAAA4imB8LDrGAAAAziIYH8ujYwwAAOAqgvGx6BgDAAA4i2B8rIQnU0AwBgAAcBHB+FhsvgMAAHAWwfgIay2jFAAAAA4jGPfqTEjWSgV0jAEAAFxEMO6V8FKveXSMAQAAXEQw7pWIp17pGAMAADiJYNzLS3WMDZvvAAAAnEQw7tXbMWbzHQAAgJMIxr16Z4w5xxgAAMBJBOMjbG8wpmMMAADgJIJxr75RCmaMAQAAXEQw7tU3SkEwBgAAcBHBuJcXl8JhKZKT7UoAAACQBQTjXom4lF8oY0y2KwEAAEAWEIx7JTw23gEAADiMYHyE9QjGAAAALiMY9+okGAMAALiMYNzLi3MiBQAAgMMIxr0ScRnOMAYAAHAWwbgXm+8AAACcRjDu5XncegcAAOAwgrEk6/tsvgMAAHAcwViSOhOp1wKCMQAAgKsIxlJqvlhilAIAAMBhBGMpdR20xCgFAACAwwjGUuoMY0mGc4wBAACcRTCWjo5S5NExBgAAcBXBWDo6SkHHGAAAwFkEY0m2b/MdHWMAAABXEYyl1OUeEqdSAAAAOIxgLHEqBQAAAAjGklKb7yIRmZycbFcCAACALCEYS6mOMWMUAAAATiMYS6mOMWMUAAAATiMYS7IeHWMAAADXZRSMt23bpp/85Cf60Y9+pOeee27I5zZv3qzbbrtNH3/88YgVGIiEJxXQMQYAAHBZ2mDs+75Wrlypn/70p3riiSf01ltv6YsvvhjwnOd5euWVV3TOOeeMSqGjKuHRMQYAAHBc2mC8d+9eTZs2TVOnTlUkEtHixYtVX18/4LnVq1frpptuUs54PNnBi8swYwwAAOC0tMG4paVF0Wi07300GlVLS0u/Z/bt26dYLKbLL7985CsMQieb7wAAAFwXSfeAtXbA54wxfR/7vq+nn35aP/jBD9L+sLq6OtXV1UmSHnvsMZWXl59IrSMmEon0+9kHEp4KyqKalKV6MLqOX29MbKy3W1hvt7De7gl6zdMG42g0qubm5r73zc3NKi0t7XufSCT0+eef65//+Z8lSQcPHtS//du/6R/+4R9UWVnZ759VW1ur2travvexWOyU/wecizxJ5QAADStJREFUjPLy8r6fbf0eqTMhz5c6s1QPRtex642Jj/V2C+vtFtbbPSO15tOnT8/oubTBuLKyUo2NjWpqalJZWZk2btyoH//4x31fLyws1MqVK/veP/zww/rLv/zLAaF4zEp4qdcCNt8BAAC4LG0wDofDuvvuu/Xoo4/K931VV1eroqJCq1evVmVlpaqqqoKoc/T0BmNmjAEAAJyWNhhL0vz58zV//vx+n7v99tsHffbhhx8+5aIC5fUGYzrGAAAALuPmu0RckmS44AMAAMBpBGNGKQAAACCCcV/HmFEKAAAAtzkfjC2nUgAAAEAEY8nr7RgzSgEAAOAygnHvKEUewRgAAMBlBOOEJ+XkykQyOrkOAAAAExTB2PMYowAAAADBWAmCMQAAAAjGsok4J1IAAACAYJzqGBOMAQAAXEcwTsQZpQAAAADBWF5cho4xAACA8wjGbL4DAACACMapYFxAMAYAAHCd08HYJpNSdxeb7wAAAOB2MFanl3pllAIAAMB5bgdjL5565RxjAAAA57kdjBOpjrGhYwwAAOA8x4PxkY4xM8YAAADOczwYM2MMAACAFKeDsfV6gzEdYwAAANc5HYz7Rik4xxgAAMB5jgdjRikAAACQ4nYw7j2ujWAMAADgPLeDcacn5ebJhMLZrgQAAABZ5nYw9uJc7gEAAABJrgfjhMeJFAAAAJDkeDC2CY/5YgAAAEhyPBgrEScYAwAAQJLrwdijYwwAAIAUt4NxIi7D5jsAAADI+WDM5jsAAACkOB6MmTEGAABAirPB2HZ3S8kkwRgAAACSHA7GSnipV0YpAAAAIKeDcTz1WkDHGAAAAE4H41TH2DBKAQAAALkcjL0jHWNGKQAAACCXg3HnkRljzjEGAACAHA7Gtq9jzCgFAAAAHA7GnEoBAACAYzkcjOkYAwAA4Ch3g7F3pGOcl5/dOgAAADAmuBuME56UXyATcvePAAAAAEe5mwoTccYoAAAA0MfhYOyx8Q4AAAB9nA3Glo4xAAAAjuFsMJYX53IPAAAA9HE3GCc8KY+OMQAAAFKcDsamgGAMAACAFKeDMZvvAAAA0MvJYGyt5bg2AAAA9ONkMFZ3l9TTw+Y7AAAA9HEyGNv44dQHdIwBAABwhJPB2PfiqQ+YMQYAAMARTgZj66U6xpxKAQAAgF5uBuP4kY4x5xgDAADgCDeDce8oBZvvAAAAcISTwdj32HwHAACA/pwMxpbNdwAAADiOm8GY49oAAABwHDeDsXdYMkbKy892KQAAABgjHA3GcSm/UMaYbJcCAACAMcLJYOzHDzNGAQAAgH6cDMapjjHBGAAAAEc5GozpGAMAAKA/N4Nx/DCXewAAAKAfJ4Oxf2TzHQAAANDLyWBsvbgMoxQAAAA4hrPBmFEKAAAAHMu5YGytTW2+y6NjDAAAgKOcC8bq6pJ8XyogGAMAAOAo94JxIp56ZcYYAAAAx3AvGHu9wZgZYwAAABzlXjDu9CRJhs13AAAAOIZ7wdhjlAIAAAADuReME6mOMaMUAAAAOJZzwdiy+Q4AAACDcC4YyzvSMea4NgAAABzDvWDMKAUAAAAG4WAwjkuhsJSTm+1KAAAAMIY4GIw9mcJCGWOyXQkAAADGEPeCsRfnDGMAAAAM4Fwwtp2eQgVF2S4DAAAAY4xzwVheXKaQYAwAAID+3AvGCY9RCgAAAAzgaDCmYwwAAID+3AvGbL4DAADAINwLxp2eQswYAwAA4DhOBWPr+6lRCm69AwAAwHGcCsbqSkjWcioFAAAABnArGCc8SZIppGMMAACA/twKxt6RYMzmOwAAABzHrWB8pGPMzXcAAAA4nlvB2O+RSspkiidnuxIAAACMMZFsFxAkU3m+wj9/Srnl5VIslu1yAAAAMIa41TEGAAAAhkAwBgAAAEQwBgAAACRlOGO8bds2rVq1Sr7vq6amRjfffHO/r7/22mt69dVXFQqFlJ+fr3vuuUczZswYlYIBAACA0ZA2GPu+r5UrV+qBBx5QNBrV/fffr6qqqn7Bd8mSJbruuuskSVu2bNHTTz+tn/3sZ6NXNQAAADDC0o5S7N27V9OmTdPUqVMViUS0ePFi1dfX93um8Jib5BKJhIwxI18pAAAAMIrSdoxbWloUjUb73kejUe3Zs2fAc2vXrtWaNWuUTCb14IMPjmyVAAAAwChLG4yttQM+N1hHeMWKFVqxYoXefPNN/fa3v9Xf/u3fDnimrq5OdXV1kqTHHntM5eXlJ1PzKYtEIln72Qge6+0W1tstrLdbWG/3BL3maYNxNBpVc3Nz3/vm5maVlpYO+fzixYv161//etCv1dbWqra2tu99LEuXbJSXl2ftZyN4rLdbWG+3sN5uYb3dM1JrPn369IyeSztjXFlZqcbGRjU1NSmZTGrjxo2qqqrq90xjY2Pfx1u3btUZZ5xxguUCAAAA2ZW2YxwOh3X33Xfr0Ucfle/7qq6uVkVFhVavXq3KykpVVVVp7dq12r59u8LhsIqLi/XDH/4wiNoBAACAEZPROcbz58/X/Pnz+33u9ttv7/v4r//6r0e2KgAAACBg3HwHAAAAiGAMAAAASCIYAwAAAJIIxgAAAIAkgjEAAAAgiWAMAAAASCIYAwAAAJIIxgAAAIAkgjEAAAAgiWAMAAAASCIYAwAAAJIIxgAAAIAkgjEAAAAgiWAMAAAASCIYAwAAAJIIxgAAAIAkyVhrbbaLAAAAALLNyY7xfffdl+0SECDW2y2st1tYb7ew3u4Jes2dDMYAAADA8QjGAAAAgKTwww8//HC2i8iG2bNnZ7sEBIj1dgvr7RbW2y2st3uCXHM23wEAAABilAIAAACQJEWyXUCQtm3bplWrVsn3fdXU1Ojmm2/OdkkYYf/xH/+hrVu3qqSkRI8//rgk6dChQ3riiSf09ddf67TTTtO9996r4uLiLFeKUxWLxfTkk0/q4MGDMsaotrZWN9xwA+s9gXV1demhhx5SMplUT0+PFi1apNtuu01NTU36xS9+oUOHDmnWrFn60Y9+pEjEqf+8TWi+7+u+++5TWVmZ7rvvPtZ7AvvhD3+o/Px8hUIhhcNhPfbYY4H/Tndmxtj3ff3rv/6rfvazn+k73/mOVq1apblz52ry5MnZLg0jqKioSNXV1aqvr9fy5cslSc8884wqKip07733qrW1VR988IEuueSSLFeKU9XZ2alzzz1X3/ve93T11VfrV7/6lS6++GKtXbuW9Z6gQqGQlixZohtuuEE1NTX67//+b1VUVOg3v/mNqqurdc8992j79u1qbW1VZWVltsvFCFmzZo2SyaSSyaSWLFmiX/3qV6z3BPXyyy/rkUce0Y033qja2lpJwf833JlRir1792ratGmaOnWqIpGIFi9erPr6+myXhRE2d+7cAX+TrK+v19KlSyVJS5cuZd0niNLS0r4NGQUFBTrzzDPV0tLCek9gxhjl5+dLknp6etTT0yNjjHbu3KlFixZJkpYtW8aaTyDNzc3aunWrampqJEnWWtbbMUH/Tnfm/3toaWlRNBrtex+NRrVnz54sVoSgtLW1qbS0VFIqTLW3t2e5Ioy0pqYm7du3T3PmzGG9Jzjf9/WP//iP2r9/v5YvX66pU6eqsLBQ4XBYklRWVqaWlpYsV4mR8tRTT+nOO++U53mSpI6ODtZ7gnv00UclSddee61qa2sD/53uTDAe7PANY0wWKgEwkhKJhB5//HH91V/9lQoLC7NdDkZZKBTSv//7v+vw4cP6+c9/ri+//DLbJWGUvPvuuyopKdHs2bO1c+fObJeDADzyyCMqKytTW1ub/uVf/kXTp08PvAZngnE0GlVzc3Pf++bm5r6/gWBiKykpUWtrq0pLS9Xa2spc+QSSTCb1+OOP66qrrtLChQslsd6uKCoq0ty5c7Vnzx7F43H19PQoHA6rpaVFZWVl2S4PI6ChoUFbtmzRe++9p66uLnmep6eeeor1nsB617KkpEQLFizQ3r17A/+d7syMcWVlpRobG9XU1KRkMqmNGzeqqqoq22UhAFVVVdqwYYMkacOGDVqwYEGWK8JIsNbql7/8pc4880x961vf6vs86z1xtbe36/Dhw5JSJ1Rs375dZ555pi688EJt3rxZkrR+/Xp+t08Qd9xxh375y1/qySef1N/93d/poosu0o9//GPWe4JKJBJ9IzOJREIffPCBZs6cGfjvdKcu+Ni6dauefvpp+b6v6upq3XLLLdkuCSPsF7/4hT788EN1dHSopKREt912mxYsWKAnnnhCsVhM5eXl+vu//3uO75oAdu3apQcffFAzZ87sG4v63ve+p3POOYf1nqA+/fRTPfnkk/J9X9ZaXXHFFfrud7+rAwcODDi+KycnJ9vlYgTt3LlTL774ou677z7We4I6cOCAfv7zn0tKba5dsmSJbrnlFnV0dAT6O92pYAwAAAAMxZlRCgAAAGA4BGMAAABABGMAAABAEsEYAAAAkEQwBgAAACQRjAEAAABJBGMAAABAEsEYAAAAkCT9f7l0NN1x2peFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x864 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot accuracies\n",
    "import pandas as pd\n",
    "hist = pd.DataFrame(log.history)\n",
    "# print(hist)\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use(\"ggplot\")\n",
    "plt.figure(figsize=(12,12))\n",
    "plt.plot(hist[\"crf_viterbi_accuracy\"], label='Training accuracy')\n",
    "plt.plot(hist[\"val_crf_viterbi_accuracy\"], label='Validation accuracy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "258/258 [==============================] - 0s 753us/step\n",
      "F1-score: 81.6%\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          E       0.86      0.91      0.89       918\n",
      "          P       0.64      0.56      0.60       323\n",
      "\n",
      "avg / total       0.80      0.82      0.81      1241\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# evaluate model on the test set\n",
    "test_pred = model.predict(X_test, verbose=1)\n",
    "\n",
    "pred_labels = [[tags[np.argmax(p)] for p in pred_i] for pred_i in test_pred]\n",
    "test_labels = [[tags[np.argmax(p)] for p in pred_i] for pred_i in y_test]\n",
    "\n",
    "from seqeval.metrics import precision_score, recall_score, f1_score, classification_report\n",
    "print(\"F1-score: {:.1%}\".format(f1_score(test_labels, pred_labels, average='weighted')))\n",
    "print(classification_report(test_labels, pred_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word           ||True ||Pred\n",
      "==============================\n",
      "which          :     0 0\n",
      "scientist      :     1 1\n",
      "was            :     0 0\n",
      "advised        :     2 2\n",
      "by             :     0 0\n",
      "john           :     1 1\n",
      "robert         :     1 1\n",
      "woodyard       :     1 1\n"
     ]
    }
   ],
   "source": [
    "# show sample test prediction\n",
    "i = 15\n",
    "p = model.predict(np.array([X_test[i]]))\n",
    "p = np.argmax(p, axis=-1)\n",
    "true = np.argmax(y_test[i], -1)\n",
    "print(\"{:15}||{:5}||{}\".format(\"Word\", \"True\", \"Pred\"))\n",
    "print(30 * \"=\")\n",
    "for w, t, pred in zip(X_test[i], true, p[0]):\n",
    "    if w != 0:\n",
    "        print(\"{:15}: {:5} {}\".format(words[w-1], t, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model\n",
    "from keras_contrib.utils import save_load_utils\n",
    "with open('model/' + modelname + '.json', 'w') as f:\n",
    "    f.write(model.to_json())\n",
    "\n",
    "# save weights\n",
    "save_load_utils.save_all_weights(model, 'model/'+modelname+'.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Complex questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', 'E1', 'E1', '', 'P1', 'P1', 'E1', 'E1']\n",
      "[0, 1, 1, 0, 2, 2, 1, 1]\n",
      "[0, 1, 1, 0, 1, 1, 1, 1]\n",
      "Training on 2316 samples testing on 258 samples\n"
     ]
    }
   ],
   "source": [
    "modelname = 'complex_questions'\n",
    "\n",
    "# split dataset into training and testing subsets\n",
    "test_size = 0.1\n",
    "from sklearn.model_selection import train_test_split\n",
    "# fix random seed\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, question_types, test_size=test_size,\n",
    "                                                    stratify=question_types, random_state=103232)\n",
    "print(\"Training on %d samples testing on %d samples\" % (len(X_train), len(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_20 (InputLayer)        (None, 24)                0         \n",
      "_________________________________________________________________\n",
      "embedding_20 (Embedding)     (None, 24, 100)           384400    \n",
      "_________________________________________________________________\n",
      "bidirectional_20 (Bidirectio (None, 100)               60400     \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 444,901\n",
      "Trainable params: 60,501\n",
      "Non-trainable params: 384,400\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# predict question type\n",
    "from keras.models import Model, Input\n",
    "from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional\n",
    "from keras_contrib.layers import CRF\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "def build_model(model_settings):\n",
    "    # architecture\n",
    "    input = Input(shape=(model_settings['max_len'],))\n",
    "    model = Embedding(input_dim=model_settings['n_words']+1, output_dim=model_settings['emb_dim'],\n",
    "                      weights=[model_settings['embeddings']],\n",
    "                      input_length=model_settings['max_len'], mask_zero=True, trainable=False)(input)\n",
    "    model = Bidirectional(LSTM(units=50, return_sequences=False,\n",
    "                               recurrent_dropout=0.1))(model)  # variational biLSTM\n",
    "    out = Dense(1, kernel_initializer='normal', activation='sigmoid')(model)  # output\n",
    "    model = Model(input, out)\n",
    "    model.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "# load model settings\n",
    "import pickle as pkl\n",
    "with open('%s_%s.pkl'%(dataset_name, embeddings_choice), 'rb') as f:\n",
    "    model_settings = pkl.load(f)\n",
    "model = build_model(model_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2084 samples, validate on 232 samples\n",
      "Epoch 1/50\n",
      "2084/2084 [==============================] - 3s 1ms/step - loss: 1.6325 - acc: 0.1032 - val_loss: 1.5620 - val_acc: 0.0948\n",
      "Epoch 2/50\n",
      "2084/2084 [==============================] - 3s 1ms/step - loss: 1.4797 - acc: 0.1032 - val_loss: 1.4364 - val_acc: 0.0948\n",
      "Epoch 3/50\n",
      "2084/2084 [==============================] - 3s 1ms/step - loss: 1.3917 - acc: 0.1032 - val_loss: 1.3599 - val_acc: 0.0948\n",
      "Epoch 4/50\n",
      "2084/2084 [==============================] - 3s 1ms/step - loss: 1.3411 - acc: 0.1032 - val_loss: 1.3176 - val_acc: 0.0948\n",
      "Epoch 5/50\n",
      "2084/2084 [==============================] - 3s 1ms/step - loss: 1.3137 - acc: 0.1032 - val_loss: 1.2882 - val_acc: 0.0948\n",
      "\n",
      "Epoch 00005: val_loss improved from inf to 1.28823, saving model to checkpoints/_complex_questions05-0.09.h5\n",
      "Epoch 6/50\n",
      "2084/2084 [==============================] - 3s 1ms/step - loss: 1.2929 - acc: 0.1032 - val_loss: 1.2683 - val_acc: 0.0948\n",
      "Epoch 00006: early stopping\n"
     ]
    }
   ],
   "source": [
    "# callbacks\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping, TerminateOnNaN, ModelCheckpoint\n",
    "cb_redlr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n",
    "cb_early = EarlyStopping(monitor='val_acc', min_delta=0, patience=5, verbose=1)\n",
    "cb_chkpt = ModelCheckpoint('checkpoints/_'+modelname+'{epoch:02d}-{val_acc:.2f}.h5',\n",
    "                           verbose=1, save_best_only=True, save_weights_only=True, period=5)\n",
    "\n",
    "callbacks_list=[cb_redlr, cb_early, cb_chkpt]\n",
    "\n",
    "# start training: weight the classes proportional\n",
    "log = model.fit(X_train, np.array(y_train), batch_size=32, epochs=50,\n",
    "                callbacks=callbacks_list,\n",
    "                validation_split=0.1, verbose=1, class_weight={0: 1., 1: 9.})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "258/258 [==============================] - 0s 496us/step\n",
      "F1-score: 1.8%\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00       232\n",
      "          1       0.10      1.00      0.18        26\n",
      "\n",
      "avg / total       0.01      0.10      0.02       258\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zola/anaconda3/envs/tf36/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/zola/anaconda3/envs/tf36/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "# evaluate\n",
    "# np.set_printoptions(precision=4, suppress=True)\n",
    "# eval_results = model.evaluate(X_test, y_test, verbose=0) \n",
    "# print(\"\\nLoss, accuracy on test data: \")\n",
    "# print(\"%0.4f %0.2f%%\" % (eval_results[0], \\\n",
    "#   eval_results[1]*100))\n",
    "\n",
    "# evaluate model on the test set\n",
    "test_pred = model.predict(X_test, verbose=1)\n",
    "y_pred = [1 if pred > 0.5 else 0 for pred in test_pred]\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, classification_report\n",
    "print(\"F1-score: {:.1%}\".format(f1_score(y_test, y_pred, average='weighted')))\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "who has his resting place at plymouth vermont and office at massachusetts house of representatives\n",
      "0.07\n"
     ]
    }
   ],
   "source": [
    "# show sample test prediction\n",
    "i = 30\n",
    "p = model.predict(np.array([X_test[i]]))\n",
    "q = \" \".join([words[w-1] for w in X_test[i] if w != 0])\n",
    "print(q)\n",
    "print(\"%.2f\"%p[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf36",
   "language": "python",
   "name": "tf36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
